@crossdelta/infrastructure 0.5.4 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/helpers/index.d.ts +2 -0
- package/dist/helpers/stream-job.d.ts +68 -0
- package/dist/helpers/stream-setup.d.ts +41 -0
- package/dist/index.cjs +188 -21
- package/dist/index.js +188 -21
- package/package.json +2 -2
package/dist/helpers/index.d.ts
CHANGED
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* NATS JetStream Stream Materialization via K8s Job
|
|
3
|
+
*
|
|
4
|
+
* Creates streams inside the cluster using a K8s Job with nats-box.
|
|
5
|
+
* This solves the problem that Pulumi runs outside the cluster
|
|
6
|
+
* and cannot reach cluster-internal NATS URLs.
|
|
7
|
+
*
|
|
8
|
+
* Architecture:
|
|
9
|
+
* - Pure functions (stream-setup.ts) generate shell scripts
|
|
10
|
+
* - This file creates Pulumi K8s resources (ConfigMap + Job)
|
|
11
|
+
* - nats-box image provides the `nats` CLI (no custom image needed)
|
|
12
|
+
*
|
|
13
|
+
* @module @crossdelta/infrastructure/lib/helpers/stream-job
|
|
14
|
+
*/
|
|
15
|
+
import type { Provider } from '@pulumi/kubernetes';
|
|
16
|
+
import * as k8s from '@pulumi/kubernetes';
|
|
17
|
+
import type { Output } from '@pulumi/pulumi';
|
|
18
|
+
import type { StreamPolicy } from './deploy-streams';
|
|
19
|
+
import { type ResolvedStream } from './stream-setup';
|
|
20
|
+
export type { ResolvedStream };
|
|
21
|
+
export { computeConfigHash, generateSetupScript, msToNatsDuration, resolveStreams } from './stream-setup';
|
|
22
|
+
export interface MaterializeStreamsConfig {
|
|
23
|
+
/** NATS connection URL (cluster-internal) */
|
|
24
|
+
natsUrl: Output<string>;
|
|
25
|
+
/** NATS auth credentials (optional) */
|
|
26
|
+
credentials?: {
|
|
27
|
+
user?: Output<string>;
|
|
28
|
+
password?: Output<string>;
|
|
29
|
+
};
|
|
30
|
+
/** Contracts module (e.g., `import * as contracts from '@my-org/contracts'`) */
|
|
31
|
+
contracts: Record<string, unknown>;
|
|
32
|
+
/** Stream-specific policy overrides */
|
|
33
|
+
policies?: Record<string, Partial<StreamPolicy>>;
|
|
34
|
+
/** Default policy for all streams */
|
|
35
|
+
defaults?: Partial<StreamPolicy>;
|
|
36
|
+
/** nats-box image override (default: natsio/nats-box:0.14.5) */
|
|
37
|
+
image?: string;
|
|
38
|
+
/** Job TTL after completion in seconds (default: 300) */
|
|
39
|
+
ttlAfterFinished?: number;
|
|
40
|
+
/** Max retry attempts (default: 3) */
|
|
41
|
+
backoffLimit?: number;
|
|
42
|
+
}
|
|
43
|
+
/**
|
|
44
|
+
* Materialize NATS JetStream streams via a K8s Job.
|
|
45
|
+
*
|
|
46
|
+
* Creates a ConfigMap with a setup script and a Job that runs it
|
|
47
|
+
* inside the cluster where NATS is reachable. Uses nats-box image
|
|
48
|
+
* (no custom image build needed).
|
|
49
|
+
*
|
|
50
|
+
* Validation runs at Pulumi plan time (outside cluster).
|
|
51
|
+
* Materialization runs at deploy time (inside cluster via K8s Job).
|
|
52
|
+
*
|
|
53
|
+
* @example
|
|
54
|
+
* ```typescript
|
|
55
|
+
* import { materializeStreams } from '@crossdelta/infrastructure'
|
|
56
|
+
* import * as contracts from '@my-org/contracts'
|
|
57
|
+
* import { STREAM_POLICIES, DEFAULT_POLICY } from '@my-org/contracts'
|
|
58
|
+
*
|
|
59
|
+
* materializeStreams(provider, 'my-namespace', {
|
|
60
|
+
* natsUrl: runtime.natsUrl,
|
|
61
|
+
* credentials: { user: natsUser, password: natsPassword },
|
|
62
|
+
* contracts,
|
|
63
|
+
* policies: STREAM_POLICIES,
|
|
64
|
+
* defaults: DEFAULT_POLICY,
|
|
65
|
+
* })
|
|
66
|
+
* ```
|
|
67
|
+
*/
|
|
68
|
+
export declare const materializeStreams: (provider: Provider, namespace: string, config: MaterializeStreamsConfig) => k8s.batch.v1.Job;
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Stream Setup Script Generator
|
|
3
|
+
*
|
|
4
|
+
* Pure functions for generating NATS JetStream setup scripts.
|
|
5
|
+
* No Pulumi or K8s dependencies — fully testable with Bun.
|
|
6
|
+
*
|
|
7
|
+
* Used by:
|
|
8
|
+
* - `stream-job.ts` (K8s Job materialization)
|
|
9
|
+
* - Tests (direct import)
|
|
10
|
+
*
|
|
11
|
+
* @module @crossdelta/infrastructure/lib/helpers/stream-setup
|
|
12
|
+
*/
|
|
13
|
+
import type { StreamPolicy } from './deploy-streams';
|
|
14
|
+
import type { StreamDefinition } from './streams';
|
|
15
|
+
export interface ResolvedStream {
|
|
16
|
+
stream: string;
|
|
17
|
+
subjects: string[];
|
|
18
|
+
retention: string;
|
|
19
|
+
storage: string;
|
|
20
|
+
maxAge: string;
|
|
21
|
+
replicas: number;
|
|
22
|
+
maxMsgSize: number;
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* Convert milliseconds to NATS CLI duration format (e.g., "168h0m0s")
|
|
26
|
+
*/
|
|
27
|
+
export declare const msToNatsDuration: (ms: number) => string;
|
|
28
|
+
/**
|
|
29
|
+
* Apply policies and produce resolved stream configs (pure)
|
|
30
|
+
*/
|
|
31
|
+
export declare const resolveStreams: (definitions: StreamDefinition[], policies?: Record<string, Partial<StreamPolicy>>, defaults?: Partial<StreamPolicy>) => ResolvedStream[];
|
|
32
|
+
/**
|
|
33
|
+
* Generate a shell script that creates/updates NATS JetStream streams.
|
|
34
|
+
* Uses `nats` CLI from nats-box. Idempotent: creates if missing, updates if exists.
|
|
35
|
+
*/
|
|
36
|
+
export declare const generateSetupScript: (streams: ResolvedStream[]) => string;
|
|
37
|
+
/**
|
|
38
|
+
* Compute a short hash from stream config for change detection.
|
|
39
|
+
* When config changes, the Job name changes → Pulumi recreates it.
|
|
40
|
+
*/
|
|
41
|
+
export declare const computeConfigHash: (streams: ResolvedStream[]) => string;
|
package/dist/index.cjs
CHANGED
|
@@ -44,8 +44,11 @@ var exports_lib = {};
|
|
|
44
44
|
__export(exports_lib, {
|
|
45
45
|
validateStreamDefinitionsPure: () => validateStreamDefinitionsPure,
|
|
46
46
|
validateStreamDefinitions: () => validateStreamDefinitions,
|
|
47
|
+
resolveStreams: () => resolveStreams,
|
|
47
48
|
registerRuntime: () => registerRuntime,
|
|
48
49
|
ports: () => ports,
|
|
50
|
+
msToNatsDuration: () => msToNatsDuration,
|
|
51
|
+
materializeStreams: () => materializeStreams,
|
|
49
52
|
hasPublicPort: () => hasPublicPort,
|
|
50
53
|
getStreamNames: () => getStreamNames,
|
|
51
54
|
getStreamDefinition: () => getStreamDefinition,
|
|
@@ -53,6 +56,7 @@ __export(exports_lib, {
|
|
|
53
56
|
getPublicPorts: () => getPublicPorts,
|
|
54
57
|
getPrimaryPort: () => getPrimaryPort,
|
|
55
58
|
getAllPorts: () => getAllPorts,
|
|
59
|
+
generateSetupScript: () => generateSetupScript,
|
|
56
60
|
generateLocalSetupScript: () => generateLocalSetupScript,
|
|
57
61
|
generateKubectlApplyCommand: () => generateKubectlApplyCommand,
|
|
58
62
|
generateK3dDeleteCommand: () => generateK3dDeleteCommand,
|
|
@@ -81,6 +85,7 @@ __export(exports_lib, {
|
|
|
81
85
|
createImagePullSecret: () => createImagePullSecret,
|
|
82
86
|
createDOKSCluster: () => createDOKSCluster,
|
|
83
87
|
convertToComposeService: () => convertToComposeService,
|
|
88
|
+
computeConfigHash: () => computeConfigHash,
|
|
84
89
|
collectStreamDefinitions: () => collectStreamDefinitions,
|
|
85
90
|
buildOtelEnv: () => buildOtelEnv,
|
|
86
91
|
buildNatsUrl: () => buildNatsUrl,
|
|
@@ -660,20 +665,16 @@ var collectStreamDefinitions = (contractsModule) => {
|
|
|
660
665
|
};
|
|
661
666
|
var getStreamDefinition = (contractsModule, streamName) => collectStreamDefinitions(contractsModule).find((s) => s.stream === streamName);
|
|
662
667
|
var getStreamNames = (contractsModule) => collectStreamDefinitions(contractsModule).map((s) => s.stream);
|
|
663
|
-
var buildSubjectMap = (streams) => streams.reduce((acc, { stream, subjects }) => {
|
|
664
|
-
for (const subject of subjects) {
|
|
665
|
-
acc.set(subject, stream);
|
|
666
|
-
}
|
|
667
|
-
return acc;
|
|
668
|
-
}, new Map);
|
|
669
668
|
var findDuplicateSubjects = (streams) => {
|
|
670
|
-
const
|
|
669
|
+
const seen = new Map;
|
|
671
670
|
const duplicates = new Set;
|
|
672
|
-
for (const { subjects } of streams) {
|
|
671
|
+
for (const { stream, subjects } of streams) {
|
|
673
672
|
for (const subject of subjects) {
|
|
674
|
-
const existingStream =
|
|
675
|
-
if (existingStream) {
|
|
673
|
+
const existingStream = seen.get(subject);
|
|
674
|
+
if (existingStream !== undefined && existingStream !== stream) {
|
|
676
675
|
duplicates.add(subject);
|
|
676
|
+
} else {
|
|
677
|
+
seen.set(subject, stream);
|
|
677
678
|
}
|
|
678
679
|
}
|
|
679
680
|
}
|
|
@@ -937,9 +938,175 @@ var buildOtelEnv = (serviceName, config) => {
|
|
|
937
938
|
}
|
|
938
939
|
return env;
|
|
939
940
|
};
|
|
941
|
+
// lib/helpers/stream-job.ts
|
|
942
|
+
var k8s4 = __toESM(require("@pulumi/kubernetes"));
|
|
943
|
+
|
|
944
|
+
// lib/helpers/stream-setup.ts
|
|
945
|
+
var DEFAULT_POLICY = {
|
|
946
|
+
maxAge: 7 * 24 * 60 * 60 * 1000,
|
|
947
|
+
storage: "file",
|
|
948
|
+
replicas: 1,
|
|
949
|
+
maxMsgSize: 1024 * 1024
|
|
950
|
+
};
|
|
951
|
+
var msToNatsDuration = (ms) => {
|
|
952
|
+
const totalSeconds = Math.floor(ms / 1000);
|
|
953
|
+
const hours = Math.floor(totalSeconds / 3600);
|
|
954
|
+
const minutes = Math.floor(totalSeconds % 3600 / 60);
|
|
955
|
+
const seconds = totalSeconds % 60;
|
|
956
|
+
return `${hours}h${minutes}m${seconds}s`;
|
|
957
|
+
};
|
|
958
|
+
var resolveStreams = (definitions, policies = {}, defaults = DEFAULT_POLICY) => definitions.map(({ stream, subjects }) => {
|
|
959
|
+
const merged = { ...DEFAULT_POLICY, ...defaults, ...policies[stream] };
|
|
960
|
+
return {
|
|
961
|
+
stream,
|
|
962
|
+
subjects,
|
|
963
|
+
retention: "limits",
|
|
964
|
+
storage: merged.storage ?? "file",
|
|
965
|
+
maxAge: msToNatsDuration(merged.maxAge ?? 7 * 24 * 60 * 60 * 1000),
|
|
966
|
+
replicas: merged.replicas ?? 1,
|
|
967
|
+
maxMsgSize: merged.maxMsgSize ?? 1024 * 1024
|
|
968
|
+
};
|
|
969
|
+
});
|
|
970
|
+
var generateSetupScript = (streams) => {
|
|
971
|
+
const streamCommands = streams.map((s) => {
|
|
972
|
+
const subjects = s.subjects.join(",");
|
|
973
|
+
const args = [
|
|
974
|
+
`--retention ${s.retention}`,
|
|
975
|
+
`--storage ${s.storage}`,
|
|
976
|
+
`--max-age ${s.maxAge}`,
|
|
977
|
+
`--replicas ${s.replicas}`,
|
|
978
|
+
`--max-msg-size ${s.maxMsgSize}`
|
|
979
|
+
].join(" ");
|
|
980
|
+
return [
|
|
981
|
+
`echo "Processing stream: ${s.stream} (${subjects})"`,
|
|
982
|
+
`if nats $NATS_OPTS stream info "${s.stream}" > /dev/null 2>&1; then`,
|
|
983
|
+
` echo " Updating existing stream..."`,
|
|
984
|
+
` nats $NATS_OPTS stream edit "${s.stream}" --subjects "${subjects}" ${args} --force`,
|
|
985
|
+
`else`,
|
|
986
|
+
` echo " Creating new stream..."`,
|
|
987
|
+
` nats $NATS_OPTS stream add "${s.stream}" --subjects "${subjects}" ${args} --defaults`,
|
|
988
|
+
`fi`,
|
|
989
|
+
`echo " ✅ ${s.stream} ready"`,
|
|
990
|
+
``
|
|
991
|
+
].join(`
|
|
992
|
+
`);
|
|
993
|
+
});
|
|
994
|
+
return [
|
|
995
|
+
`#!/bin/sh`,
|
|
996
|
+
`set -e`,
|
|
997
|
+
``,
|
|
998
|
+
`NATS_OPTS="--server $NATS_URL"`,
|
|
999
|
+
`if [ -n "$NATS_USER" ] && [ -n "$NATS_PASSWORD" ]; then`,
|
|
1000
|
+
` NATS_OPTS="$NATS_OPTS --user $NATS_USER --password $NATS_PASSWORD"`,
|
|
1001
|
+
`fi`,
|
|
1002
|
+
``,
|
|
1003
|
+
`echo "\uD83D\uDE80 Materializing ${streams.length} stream(s)..."`,
|
|
1004
|
+
`echo ""`,
|
|
1005
|
+
``,
|
|
1006
|
+
...streamCommands,
|
|
1007
|
+
`echo ""`,
|
|
1008
|
+
`echo "✅ All streams materialized successfully"`
|
|
1009
|
+
].join(`
|
|
1010
|
+
`);
|
|
1011
|
+
};
|
|
1012
|
+
var computeConfigHash = (streams) => {
|
|
1013
|
+
const data = JSON.stringify(streams);
|
|
1014
|
+
let hash = 0;
|
|
1015
|
+
for (let i = 0;i < data.length; i++) {
|
|
1016
|
+
const char = data.charCodeAt(i);
|
|
1017
|
+
hash = (hash << 5) - hash + char | 0;
|
|
1018
|
+
}
|
|
1019
|
+
return Math.abs(hash).toString(36).slice(0, 8);
|
|
1020
|
+
};
|
|
1021
|
+
|
|
1022
|
+
// lib/helpers/stream-job.ts
|
|
1023
|
+
var DEFAULT_POLICY2 = {
|
|
1024
|
+
maxAge: 7 * 24 * 60 * 60 * 1000,
|
|
1025
|
+
storage: "file",
|
|
1026
|
+
replicas: 1,
|
|
1027
|
+
maxMsgSize: 1024 * 1024
|
|
1028
|
+
};
|
|
1029
|
+
var NATS_BOX_IMAGE = "natsio/nats-box:0.14.5";
|
|
1030
|
+
var materializeStreams = (provider, namespace, config) => {
|
|
1031
|
+
const {
|
|
1032
|
+
contracts,
|
|
1033
|
+
policies = {},
|
|
1034
|
+
defaults = DEFAULT_POLICY2,
|
|
1035
|
+
image = NATS_BOX_IMAGE,
|
|
1036
|
+
ttlAfterFinished = 300,
|
|
1037
|
+
backoffLimit = 3
|
|
1038
|
+
} = config;
|
|
1039
|
+
validateStreamDefinitions(contracts);
|
|
1040
|
+
const definitions = collectStreamDefinitions(contracts);
|
|
1041
|
+
const resolved = resolveStreams(definitions, policies, defaults);
|
|
1042
|
+
const script = generateSetupScript(resolved);
|
|
1043
|
+
const configHash = computeConfigHash(resolved);
|
|
1044
|
+
console.log(`\uD83D\uDCCB Collecting stream definitions from contracts...`);
|
|
1045
|
+
console.log(`✅ Found ${definitions.length} stream(s):`);
|
|
1046
|
+
for (const { stream, subjects } of definitions) {
|
|
1047
|
+
console.log(` - ${stream}: ${subjects.join(", ")}`);
|
|
1048
|
+
}
|
|
1049
|
+
const configMap = new k8s4.core.v1.ConfigMap("stream-setup-config", {
|
|
1050
|
+
metadata: {
|
|
1051
|
+
name: `stream-setup-${configHash}`,
|
|
1052
|
+
namespace,
|
|
1053
|
+
labels: { app: "stream-setup", "config-hash": configHash }
|
|
1054
|
+
},
|
|
1055
|
+
data: {
|
|
1056
|
+
"setup.sh": script,
|
|
1057
|
+
"streams.json": JSON.stringify(resolved, null, 2)
|
|
1058
|
+
}
|
|
1059
|
+
}, { provider });
|
|
1060
|
+
const envVars = [{ name: "NATS_URL", value: config.natsUrl }];
|
|
1061
|
+
if (config.credentials?.user) {
|
|
1062
|
+
envVars.push({ name: "NATS_USER", value: config.credentials.user });
|
|
1063
|
+
}
|
|
1064
|
+
if (config.credentials?.password) {
|
|
1065
|
+
envVars.push({ name: "NATS_PASSWORD", value: config.credentials.password });
|
|
1066
|
+
}
|
|
1067
|
+
const job = new k8s4.batch.v1.Job("stream-setup", {
|
|
1068
|
+
metadata: {
|
|
1069
|
+
name: `stream-setup-${configHash}`,
|
|
1070
|
+
namespace,
|
|
1071
|
+
labels: { app: "stream-setup", "config-hash": configHash }
|
|
1072
|
+
},
|
|
1073
|
+
spec: {
|
|
1074
|
+
backoffLimit,
|
|
1075
|
+
ttlSecondsAfterFinished: ttlAfterFinished,
|
|
1076
|
+
template: {
|
|
1077
|
+
metadata: {
|
|
1078
|
+
labels: { app: "stream-setup" }
|
|
1079
|
+
},
|
|
1080
|
+
spec: {
|
|
1081
|
+
restartPolicy: "Never",
|
|
1082
|
+
containers: [
|
|
1083
|
+
{
|
|
1084
|
+
name: "stream-setup",
|
|
1085
|
+
image,
|
|
1086
|
+
command: ["sh", "/config/setup.sh"],
|
|
1087
|
+
env: envVars,
|
|
1088
|
+
volumeMounts: [{ name: "config", mountPath: "/config", readOnly: true }]
|
|
1089
|
+
}
|
|
1090
|
+
],
|
|
1091
|
+
volumes: [
|
|
1092
|
+
{
|
|
1093
|
+
name: "config",
|
|
1094
|
+
configMap: { name: configMap.metadata.name }
|
|
1095
|
+
}
|
|
1096
|
+
]
|
|
1097
|
+
}
|
|
1098
|
+
}
|
|
1099
|
+
}
|
|
1100
|
+
}, {
|
|
1101
|
+
provider,
|
|
1102
|
+
dependsOn: [configMap],
|
|
1103
|
+
deleteBeforeReplace: true
|
|
1104
|
+
});
|
|
1105
|
+
return job;
|
|
1106
|
+
};
|
|
940
1107
|
// lib/runtimes/doks/cluster.ts
|
|
941
1108
|
var digitalocean = __toESM(require("@pulumi/digitalocean"));
|
|
942
|
-
var
|
|
1109
|
+
var k8s5 = __toESM(require("@pulumi/kubernetes"));
|
|
943
1110
|
var pulumi4 = __toESM(require("@pulumi/pulumi"));
|
|
944
1111
|
function createDOKSCluster(config) {
|
|
945
1112
|
const stack = pulumi4.getStack();
|
|
@@ -974,7 +1141,7 @@ function createDOKSCluster(config) {
|
|
|
974
1141
|
}
|
|
975
1142
|
return firstConfig.rawConfig;
|
|
976
1143
|
});
|
|
977
|
-
const provider = new
|
|
1144
|
+
const provider = new k8s5.Provider(`${config.name}-k8s-provider`, {
|
|
978
1145
|
kubeconfig
|
|
979
1146
|
});
|
|
980
1147
|
return {
|
|
@@ -985,7 +1152,7 @@ function createDOKSCluster(config) {
|
|
|
985
1152
|
};
|
|
986
1153
|
}
|
|
987
1154
|
function createK8sProviderFromKubeconfig(name, kubeconfig) {
|
|
988
|
-
return new
|
|
1155
|
+
return new k8s5.Provider(name, { kubeconfig });
|
|
989
1156
|
}
|
|
990
1157
|
// lib/runtimes/doks/vpc.ts
|
|
991
1158
|
var digitalocean2 = __toESM(require("@pulumi/digitalocean"));
|
|
@@ -1001,7 +1168,7 @@ function createVPC(config) {
|
|
|
1001
1168
|
});
|
|
1002
1169
|
}
|
|
1003
1170
|
// lib/runtimes/doks/workloads.ts
|
|
1004
|
-
var
|
|
1171
|
+
var k8s6 = __toESM(require("@pulumi/kubernetes"));
|
|
1005
1172
|
var pulumi6 = __toESM(require("@pulumi/pulumi"));
|
|
1006
1173
|
var normalizeK8sConfig = (config) => {
|
|
1007
1174
|
if (config.ports) {
|
|
@@ -1051,7 +1218,7 @@ var createImagePullSecret = (provider, namespace, name, config) => {
|
|
|
1051
1218
|
}
|
|
1052
1219
|
});
|
|
1053
1220
|
});
|
|
1054
|
-
return new
|
|
1221
|
+
return new k8s6.core.v1.Secret(name, {
|
|
1055
1222
|
metadata: {
|
|
1056
1223
|
name,
|
|
1057
1224
|
namespace,
|
|
@@ -1081,7 +1248,7 @@ var buildEnvVars = (config) => {
|
|
|
1081
1248
|
})) : [];
|
|
1082
1249
|
return [portEnv, ...plainEnvVars, ...secretEnvVars];
|
|
1083
1250
|
};
|
|
1084
|
-
var createServiceSecret = (provider, namespace, config, labels) => !config.secrets || Object.keys(config.secrets).length === 0 ? undefined : new
|
|
1251
|
+
var createServiceSecret = (provider, namespace, config, labels) => !config.secrets || Object.keys(config.secrets).length === 0 ? undefined : new k8s6.core.v1.Secret(`${config.name}-secret`, {
|
|
1085
1252
|
metadata: {
|
|
1086
1253
|
name: `${config.name}-secret`,
|
|
1087
1254
|
namespace,
|
|
@@ -1094,7 +1261,7 @@ var createServiceVolumes = (provider, namespace, config, labels) => {
|
|
|
1094
1261
|
if (!config.volumes) {
|
|
1095
1262
|
return { pvcs: [], volumeMounts: [], volumes: [] };
|
|
1096
1263
|
}
|
|
1097
|
-
const pvcs = config.volumes.map((vol) => new
|
|
1264
|
+
const pvcs = config.volumes.map((vol) => new k8s6.core.v1.PersistentVolumeClaim(`${config.name}-${vol.name}`, {
|
|
1098
1265
|
metadata: {
|
|
1099
1266
|
name: `${config.name}-${vol.name}`,
|
|
1100
1267
|
namespace,
|
|
@@ -1210,7 +1377,7 @@ var createServiceIngress = (provider, namespace, config, labels, service) => {
|
|
|
1210
1377
|
const allHosts = [...primaryHosts, ...additionalHosts];
|
|
1211
1378
|
const ingressRules = allHosts.length > 0 ? allHosts.map(createRule) : [createRule()];
|
|
1212
1379
|
const tlsSecretName = config.ingress.tls?.secretName ?? `${config.name}-tls`;
|
|
1213
|
-
return new
|
|
1380
|
+
return new k8s6.networking.v1.Ingress(`${config.name}-ingress`, {
|
|
1214
1381
|
metadata: {
|
|
1215
1382
|
name: config.name,
|
|
1216
1383
|
namespace,
|
|
@@ -1240,7 +1407,7 @@ var deployK8sService = (provider, namespace, config) => {
|
|
|
1240
1407
|
const { livenessProbe, readinessProbe } = buildHealthProbes(normalizedConfig);
|
|
1241
1408
|
const containerPorts = buildContainerPorts(normalizedConfig);
|
|
1242
1409
|
const servicePorts = buildServicePorts(normalizedConfig);
|
|
1243
|
-
const deployment = new
|
|
1410
|
+
const deployment = new k8s6.apps.v1.Deployment(`${normalizedConfig.name}-deployment`, {
|
|
1244
1411
|
metadata: {
|
|
1245
1412
|
name: normalizedConfig.name,
|
|
1246
1413
|
namespace,
|
|
@@ -1281,7 +1448,7 @@ var deployK8sService = (provider, namespace, config) => {
|
|
|
1281
1448
|
}
|
|
1282
1449
|
}
|
|
1283
1450
|
}, { provider, dependsOn: pvcs.length > 0 ? pvcs : undefined });
|
|
1284
|
-
const service = new
|
|
1451
|
+
const service = new k8s6.core.v1.Service(`${normalizedConfig.name}-service`, {
|
|
1285
1452
|
metadata: {
|
|
1286
1453
|
name: normalizedConfig.name,
|
|
1287
1454
|
namespace,
|
|
@@ -1314,7 +1481,7 @@ var deployK8sServices = (provider, namespace, configs, options) => configs.filte
|
|
|
1314
1481
|
results.set(config.name, deployK8sService(provider, namespace, configWithSecret));
|
|
1315
1482
|
return results;
|
|
1316
1483
|
}, new Map);
|
|
1317
|
-
var createNamespace = (provider, name, labels) => new
|
|
1484
|
+
var createNamespace = (provider, name, labels) => new k8s6.core.v1.Namespace(name, {
|
|
1318
1485
|
metadata: {
|
|
1319
1486
|
name,
|
|
1320
1487
|
labels: {
|
package/dist/index.js
CHANGED
|
@@ -571,20 +571,16 @@ var collectStreamDefinitions = (contractsModule) => {
|
|
|
571
571
|
};
|
|
572
572
|
var getStreamDefinition = (contractsModule, streamName) => collectStreamDefinitions(contractsModule).find((s) => s.stream === streamName);
|
|
573
573
|
var getStreamNames = (contractsModule) => collectStreamDefinitions(contractsModule).map((s) => s.stream);
|
|
574
|
-
var buildSubjectMap = (streams) => streams.reduce((acc, { stream, subjects }) => {
|
|
575
|
-
for (const subject of subjects) {
|
|
576
|
-
acc.set(subject, stream);
|
|
577
|
-
}
|
|
578
|
-
return acc;
|
|
579
|
-
}, new Map);
|
|
580
574
|
var findDuplicateSubjects = (streams) => {
|
|
581
|
-
const
|
|
575
|
+
const seen = new Map;
|
|
582
576
|
const duplicates = new Set;
|
|
583
|
-
for (const { subjects } of streams) {
|
|
577
|
+
for (const { stream, subjects } of streams) {
|
|
584
578
|
for (const subject of subjects) {
|
|
585
|
-
const existingStream =
|
|
586
|
-
if (existingStream) {
|
|
579
|
+
const existingStream = seen.get(subject);
|
|
580
|
+
if (existingStream !== undefined && existingStream !== stream) {
|
|
587
581
|
duplicates.add(subject);
|
|
582
|
+
} else {
|
|
583
|
+
seen.set(subject, stream);
|
|
588
584
|
}
|
|
589
585
|
}
|
|
590
586
|
}
|
|
@@ -848,9 +844,175 @@ var buildOtelEnv = (serviceName, config) => {
|
|
|
848
844
|
}
|
|
849
845
|
return env;
|
|
850
846
|
};
|
|
847
|
+
// lib/helpers/stream-job.ts
|
|
848
|
+
import * as k8s4 from "@pulumi/kubernetes";
|
|
849
|
+
|
|
850
|
+
// lib/helpers/stream-setup.ts
|
|
851
|
+
var DEFAULT_POLICY = {
|
|
852
|
+
maxAge: 7 * 24 * 60 * 60 * 1000,
|
|
853
|
+
storage: "file",
|
|
854
|
+
replicas: 1,
|
|
855
|
+
maxMsgSize: 1024 * 1024
|
|
856
|
+
};
|
|
857
|
+
var msToNatsDuration = (ms) => {
|
|
858
|
+
const totalSeconds = Math.floor(ms / 1000);
|
|
859
|
+
const hours = Math.floor(totalSeconds / 3600);
|
|
860
|
+
const minutes = Math.floor(totalSeconds % 3600 / 60);
|
|
861
|
+
const seconds = totalSeconds % 60;
|
|
862
|
+
return `${hours}h${minutes}m${seconds}s`;
|
|
863
|
+
};
|
|
864
|
+
var resolveStreams = (definitions, policies = {}, defaults = DEFAULT_POLICY) => definitions.map(({ stream, subjects }) => {
|
|
865
|
+
const merged = { ...DEFAULT_POLICY, ...defaults, ...policies[stream] };
|
|
866
|
+
return {
|
|
867
|
+
stream,
|
|
868
|
+
subjects,
|
|
869
|
+
retention: "limits",
|
|
870
|
+
storage: merged.storage ?? "file",
|
|
871
|
+
maxAge: msToNatsDuration(merged.maxAge ?? 7 * 24 * 60 * 60 * 1000),
|
|
872
|
+
replicas: merged.replicas ?? 1,
|
|
873
|
+
maxMsgSize: merged.maxMsgSize ?? 1024 * 1024
|
|
874
|
+
};
|
|
875
|
+
});
|
|
876
|
+
var generateSetupScript = (streams) => {
|
|
877
|
+
const streamCommands = streams.map((s) => {
|
|
878
|
+
const subjects = s.subjects.join(",");
|
|
879
|
+
const args = [
|
|
880
|
+
`--retention ${s.retention}`,
|
|
881
|
+
`--storage ${s.storage}`,
|
|
882
|
+
`--max-age ${s.maxAge}`,
|
|
883
|
+
`--replicas ${s.replicas}`,
|
|
884
|
+
`--max-msg-size ${s.maxMsgSize}`
|
|
885
|
+
].join(" ");
|
|
886
|
+
return [
|
|
887
|
+
`echo "Processing stream: ${s.stream} (${subjects})"`,
|
|
888
|
+
`if nats $NATS_OPTS stream info "${s.stream}" > /dev/null 2>&1; then`,
|
|
889
|
+
` echo " Updating existing stream..."`,
|
|
890
|
+
` nats $NATS_OPTS stream edit "${s.stream}" --subjects "${subjects}" ${args} --force`,
|
|
891
|
+
`else`,
|
|
892
|
+
` echo " Creating new stream..."`,
|
|
893
|
+
` nats $NATS_OPTS stream add "${s.stream}" --subjects "${subjects}" ${args} --defaults`,
|
|
894
|
+
`fi`,
|
|
895
|
+
`echo " ✅ ${s.stream} ready"`,
|
|
896
|
+
``
|
|
897
|
+
].join(`
|
|
898
|
+
`);
|
|
899
|
+
});
|
|
900
|
+
return [
|
|
901
|
+
`#!/bin/sh`,
|
|
902
|
+
`set -e`,
|
|
903
|
+
``,
|
|
904
|
+
`NATS_OPTS="--server $NATS_URL"`,
|
|
905
|
+
`if [ -n "$NATS_USER" ] && [ -n "$NATS_PASSWORD" ]; then`,
|
|
906
|
+
` NATS_OPTS="$NATS_OPTS --user $NATS_USER --password $NATS_PASSWORD"`,
|
|
907
|
+
`fi`,
|
|
908
|
+
``,
|
|
909
|
+
`echo "\uD83D\uDE80 Materializing ${streams.length} stream(s)..."`,
|
|
910
|
+
`echo ""`,
|
|
911
|
+
``,
|
|
912
|
+
...streamCommands,
|
|
913
|
+
`echo ""`,
|
|
914
|
+
`echo "✅ All streams materialized successfully"`
|
|
915
|
+
].join(`
|
|
916
|
+
`);
|
|
917
|
+
};
|
|
918
|
+
var computeConfigHash = (streams) => {
|
|
919
|
+
const data = JSON.stringify(streams);
|
|
920
|
+
let hash = 0;
|
|
921
|
+
for (let i = 0;i < data.length; i++) {
|
|
922
|
+
const char = data.charCodeAt(i);
|
|
923
|
+
hash = (hash << 5) - hash + char | 0;
|
|
924
|
+
}
|
|
925
|
+
return Math.abs(hash).toString(36).slice(0, 8);
|
|
926
|
+
};
|
|
927
|
+
|
|
928
|
+
// lib/helpers/stream-job.ts
|
|
929
|
+
var DEFAULT_POLICY2 = {
|
|
930
|
+
maxAge: 7 * 24 * 60 * 60 * 1000,
|
|
931
|
+
storage: "file",
|
|
932
|
+
replicas: 1,
|
|
933
|
+
maxMsgSize: 1024 * 1024
|
|
934
|
+
};
|
|
935
|
+
var NATS_BOX_IMAGE = "natsio/nats-box:0.14.5";
|
|
936
|
+
var materializeStreams = (provider, namespace, config) => {
|
|
937
|
+
const {
|
|
938
|
+
contracts,
|
|
939
|
+
policies = {},
|
|
940
|
+
defaults = DEFAULT_POLICY2,
|
|
941
|
+
image = NATS_BOX_IMAGE,
|
|
942
|
+
ttlAfterFinished = 300,
|
|
943
|
+
backoffLimit = 3
|
|
944
|
+
} = config;
|
|
945
|
+
validateStreamDefinitions(contracts);
|
|
946
|
+
const definitions = collectStreamDefinitions(contracts);
|
|
947
|
+
const resolved = resolveStreams(definitions, policies, defaults);
|
|
948
|
+
const script = generateSetupScript(resolved);
|
|
949
|
+
const configHash = computeConfigHash(resolved);
|
|
950
|
+
console.log(`\uD83D\uDCCB Collecting stream definitions from contracts...`);
|
|
951
|
+
console.log(`✅ Found ${definitions.length} stream(s):`);
|
|
952
|
+
for (const { stream, subjects } of definitions) {
|
|
953
|
+
console.log(` - ${stream}: ${subjects.join(", ")}`);
|
|
954
|
+
}
|
|
955
|
+
const configMap = new k8s4.core.v1.ConfigMap("stream-setup-config", {
|
|
956
|
+
metadata: {
|
|
957
|
+
name: `stream-setup-${configHash}`,
|
|
958
|
+
namespace,
|
|
959
|
+
labels: { app: "stream-setup", "config-hash": configHash }
|
|
960
|
+
},
|
|
961
|
+
data: {
|
|
962
|
+
"setup.sh": script,
|
|
963
|
+
"streams.json": JSON.stringify(resolved, null, 2)
|
|
964
|
+
}
|
|
965
|
+
}, { provider });
|
|
966
|
+
const envVars = [{ name: "NATS_URL", value: config.natsUrl }];
|
|
967
|
+
if (config.credentials?.user) {
|
|
968
|
+
envVars.push({ name: "NATS_USER", value: config.credentials.user });
|
|
969
|
+
}
|
|
970
|
+
if (config.credentials?.password) {
|
|
971
|
+
envVars.push({ name: "NATS_PASSWORD", value: config.credentials.password });
|
|
972
|
+
}
|
|
973
|
+
const job = new k8s4.batch.v1.Job("stream-setup", {
|
|
974
|
+
metadata: {
|
|
975
|
+
name: `stream-setup-${configHash}`,
|
|
976
|
+
namespace,
|
|
977
|
+
labels: { app: "stream-setup", "config-hash": configHash }
|
|
978
|
+
},
|
|
979
|
+
spec: {
|
|
980
|
+
backoffLimit,
|
|
981
|
+
ttlSecondsAfterFinished: ttlAfterFinished,
|
|
982
|
+
template: {
|
|
983
|
+
metadata: {
|
|
984
|
+
labels: { app: "stream-setup" }
|
|
985
|
+
},
|
|
986
|
+
spec: {
|
|
987
|
+
restartPolicy: "Never",
|
|
988
|
+
containers: [
|
|
989
|
+
{
|
|
990
|
+
name: "stream-setup",
|
|
991
|
+
image,
|
|
992
|
+
command: ["sh", "/config/setup.sh"],
|
|
993
|
+
env: envVars,
|
|
994
|
+
volumeMounts: [{ name: "config", mountPath: "/config", readOnly: true }]
|
|
995
|
+
}
|
|
996
|
+
],
|
|
997
|
+
volumes: [
|
|
998
|
+
{
|
|
999
|
+
name: "config",
|
|
1000
|
+
configMap: { name: configMap.metadata.name }
|
|
1001
|
+
}
|
|
1002
|
+
]
|
|
1003
|
+
}
|
|
1004
|
+
}
|
|
1005
|
+
}
|
|
1006
|
+
}, {
|
|
1007
|
+
provider,
|
|
1008
|
+
dependsOn: [configMap],
|
|
1009
|
+
deleteBeforeReplace: true
|
|
1010
|
+
});
|
|
1011
|
+
return job;
|
|
1012
|
+
};
|
|
851
1013
|
// lib/runtimes/doks/cluster.ts
|
|
852
1014
|
import * as digitalocean from "@pulumi/digitalocean";
|
|
853
|
-
import * as
|
|
1015
|
+
import * as k8s5 from "@pulumi/kubernetes";
|
|
854
1016
|
import * as pulumi4 from "@pulumi/pulumi";
|
|
855
1017
|
function createDOKSCluster(config) {
|
|
856
1018
|
const stack = pulumi4.getStack();
|
|
@@ -885,7 +1047,7 @@ function createDOKSCluster(config) {
|
|
|
885
1047
|
}
|
|
886
1048
|
return firstConfig.rawConfig;
|
|
887
1049
|
});
|
|
888
|
-
const provider = new
|
|
1050
|
+
const provider = new k8s5.Provider(`${config.name}-k8s-provider`, {
|
|
889
1051
|
kubeconfig
|
|
890
1052
|
});
|
|
891
1053
|
return {
|
|
@@ -896,7 +1058,7 @@ function createDOKSCluster(config) {
|
|
|
896
1058
|
};
|
|
897
1059
|
}
|
|
898
1060
|
function createK8sProviderFromKubeconfig(name, kubeconfig) {
|
|
899
|
-
return new
|
|
1061
|
+
return new k8s5.Provider(name, { kubeconfig });
|
|
900
1062
|
}
|
|
901
1063
|
// lib/runtimes/doks/vpc.ts
|
|
902
1064
|
import * as digitalocean2 from "@pulumi/digitalocean";
|
|
@@ -912,7 +1074,7 @@ function createVPC(config) {
|
|
|
912
1074
|
});
|
|
913
1075
|
}
|
|
914
1076
|
// lib/runtimes/doks/workloads.ts
|
|
915
|
-
import * as
|
|
1077
|
+
import * as k8s6 from "@pulumi/kubernetes";
|
|
916
1078
|
import * as pulumi6 from "@pulumi/pulumi";
|
|
917
1079
|
var normalizeK8sConfig = (config) => {
|
|
918
1080
|
if (config.ports) {
|
|
@@ -962,7 +1124,7 @@ var createImagePullSecret = (provider, namespace, name, config) => {
|
|
|
962
1124
|
}
|
|
963
1125
|
});
|
|
964
1126
|
});
|
|
965
|
-
return new
|
|
1127
|
+
return new k8s6.core.v1.Secret(name, {
|
|
966
1128
|
metadata: {
|
|
967
1129
|
name,
|
|
968
1130
|
namespace,
|
|
@@ -992,7 +1154,7 @@ var buildEnvVars = (config) => {
|
|
|
992
1154
|
})) : [];
|
|
993
1155
|
return [portEnv, ...plainEnvVars, ...secretEnvVars];
|
|
994
1156
|
};
|
|
995
|
-
var createServiceSecret = (provider, namespace, config, labels) => !config.secrets || Object.keys(config.secrets).length === 0 ? undefined : new
|
|
1157
|
+
var createServiceSecret = (provider, namespace, config, labels) => !config.secrets || Object.keys(config.secrets).length === 0 ? undefined : new k8s6.core.v1.Secret(`${config.name}-secret`, {
|
|
996
1158
|
metadata: {
|
|
997
1159
|
name: `${config.name}-secret`,
|
|
998
1160
|
namespace,
|
|
@@ -1005,7 +1167,7 @@ var createServiceVolumes = (provider, namespace, config, labels) => {
|
|
|
1005
1167
|
if (!config.volumes) {
|
|
1006
1168
|
return { pvcs: [], volumeMounts: [], volumes: [] };
|
|
1007
1169
|
}
|
|
1008
|
-
const pvcs = config.volumes.map((vol) => new
|
|
1170
|
+
const pvcs = config.volumes.map((vol) => new k8s6.core.v1.PersistentVolumeClaim(`${config.name}-${vol.name}`, {
|
|
1009
1171
|
metadata: {
|
|
1010
1172
|
name: `${config.name}-${vol.name}`,
|
|
1011
1173
|
namespace,
|
|
@@ -1121,7 +1283,7 @@ var createServiceIngress = (provider, namespace, config, labels, service) => {
|
|
|
1121
1283
|
const allHosts = [...primaryHosts, ...additionalHosts];
|
|
1122
1284
|
const ingressRules = allHosts.length > 0 ? allHosts.map(createRule) : [createRule()];
|
|
1123
1285
|
const tlsSecretName = config.ingress.tls?.secretName ?? `${config.name}-tls`;
|
|
1124
|
-
return new
|
|
1286
|
+
return new k8s6.networking.v1.Ingress(`${config.name}-ingress`, {
|
|
1125
1287
|
metadata: {
|
|
1126
1288
|
name: config.name,
|
|
1127
1289
|
namespace,
|
|
@@ -1151,7 +1313,7 @@ var deployK8sService = (provider, namespace, config) => {
|
|
|
1151
1313
|
const { livenessProbe, readinessProbe } = buildHealthProbes(normalizedConfig);
|
|
1152
1314
|
const containerPorts = buildContainerPorts(normalizedConfig);
|
|
1153
1315
|
const servicePorts = buildServicePorts(normalizedConfig);
|
|
1154
|
-
const deployment = new
|
|
1316
|
+
const deployment = new k8s6.apps.v1.Deployment(`${normalizedConfig.name}-deployment`, {
|
|
1155
1317
|
metadata: {
|
|
1156
1318
|
name: normalizedConfig.name,
|
|
1157
1319
|
namespace,
|
|
@@ -1192,7 +1354,7 @@ var deployK8sService = (provider, namespace, config) => {
|
|
|
1192
1354
|
}
|
|
1193
1355
|
}
|
|
1194
1356
|
}, { provider, dependsOn: pvcs.length > 0 ? pvcs : undefined });
|
|
1195
|
-
const service = new
|
|
1357
|
+
const service = new k8s6.core.v1.Service(`${normalizedConfig.name}-service`, {
|
|
1196
1358
|
metadata: {
|
|
1197
1359
|
name: normalizedConfig.name,
|
|
1198
1360
|
namespace,
|
|
@@ -1225,7 +1387,7 @@ var deployK8sServices = (provider, namespace, configs, options) => configs.filte
|
|
|
1225
1387
|
results.set(config.name, deployK8sService(provider, namespace, configWithSecret));
|
|
1226
1388
|
return results;
|
|
1227
1389
|
}, new Map);
|
|
1228
|
-
var createNamespace = (provider, name, labels) => new
|
|
1390
|
+
var createNamespace = (provider, name, labels) => new k8s6.core.v1.Namespace(name, {
|
|
1229
1391
|
metadata: {
|
|
1230
1392
|
name,
|
|
1231
1393
|
labels: {
|
|
@@ -1599,8 +1761,11 @@ var generateLocalSetupScript = (services, options = {}) => {
|
|
|
1599
1761
|
export {
|
|
1600
1762
|
validateStreamDefinitionsPure,
|
|
1601
1763
|
validateStreamDefinitions,
|
|
1764
|
+
resolveStreams,
|
|
1602
1765
|
registerRuntime,
|
|
1603
1766
|
ports,
|
|
1767
|
+
msToNatsDuration,
|
|
1768
|
+
materializeStreams,
|
|
1604
1769
|
hasPublicPort,
|
|
1605
1770
|
getStreamNames,
|
|
1606
1771
|
getStreamDefinition,
|
|
@@ -1608,6 +1773,7 @@ export {
|
|
|
1608
1773
|
getPublicPorts,
|
|
1609
1774
|
getPrimaryPort,
|
|
1610
1775
|
getAllPorts,
|
|
1776
|
+
generateSetupScript,
|
|
1611
1777
|
generateLocalSetupScript,
|
|
1612
1778
|
generateKubectlApplyCommand,
|
|
1613
1779
|
generateK3dDeleteCommand,
|
|
@@ -1636,6 +1802,7 @@ export {
|
|
|
1636
1802
|
createImagePullSecret,
|
|
1637
1803
|
createDOKSCluster,
|
|
1638
1804
|
convertToComposeService,
|
|
1805
|
+
computeConfigHash,
|
|
1639
1806
|
collectStreamDefinitions,
|
|
1640
1807
|
buildOtelEnv,
|
|
1641
1808
|
buildNatsUrl,
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crossdelta/infrastructure",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.6.0",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"publishConfig": {
|
|
@@ -35,7 +35,7 @@
|
|
|
35
35
|
}
|
|
36
36
|
},
|
|
37
37
|
"dependencies": {
|
|
38
|
-
"@crossdelta/cloudevents": "0.6.
|
|
38
|
+
"@crossdelta/cloudevents": "0.6.4"
|
|
39
39
|
},
|
|
40
40
|
"peerDependencies": {
|
|
41
41
|
"@pulumi/digitalocean": "^4.0.0",
|