@hatchet-dev/typescript-sdk 0.15.0 → 0.16.0-alpha.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/clients/hatchet-client/client-config.d.ts +6 -0
- package/clients/hatchet-client/client-config.js +2 -0
- package/clients/hatchet-client/hatchet-client.d.ts +2 -0
- package/clients/hatchet-client/hatchet-client.js +25 -1
- package/clients/rest/api.d.ts +3 -0
- package/clients/rest/api.js +10 -0
- package/clients/rest/generated/cloud/Api.d.ts +377 -0
- package/clients/rest/generated/cloud/Api.js +326 -0
- package/clients/rest/generated/cloud/data-contracts.d.ts +468 -0
- package/clients/rest/generated/cloud/data-contracts.js +68 -0
- package/clients/rest/generated/cloud/http-client.d.ts +41 -0
- package/clients/rest/generated/cloud/http-client.js +102 -0
- package/clients/rest/index.d.ts +2 -2
- package/clients/rest/index.js +4 -5
- package/clients/worker/compute/compute-config.d.ts +150 -0
- package/clients/worker/compute/compute-config.js +76 -0
- package/clients/worker/compute/managed-compute.d.ts +15 -0
- package/clients/worker/compute/managed-compute.js +100 -0
- package/clients/worker/worker.d.ts +10 -2
- package/clients/worker/worker.js +31 -10
- package/examples/managed-worker.d.ts +1 -0
- package/examples/managed-worker.js +65 -0
- package/package.json +4 -1
- package/protoc/dispatcher/dispatcher.d.ts +19 -0
- package/protoc/dispatcher/dispatcher.js +368 -92
- package/protoc/events/events.js +45 -23
- package/protoc/google/protobuf/timestamp.js +5 -3
- package/protoc/workflows/workflows.js +153 -77
- package/step.d.ts +117 -0
- package/step.js +2 -0
- package/util/config-loader/config-loader.js +19 -1
- package/workflow.d.ts +443 -0
package/clients/rest/index.js
CHANGED
|
@@ -22,12 +22,11 @@ var __importStar = (this && this.__importStar) || function (mod) {
|
|
|
22
22
|
__setModuleDefault(result, mod);
|
|
23
23
|
return result;
|
|
24
24
|
};
|
|
25
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
26
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
27
|
-
};
|
|
28
25
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
29
|
-
exports.Api = exports.APIContracts = void 0;
|
|
30
|
-
const api_1 =
|
|
26
|
+
exports.cloudApi = exports.CloudApi = exports.Api = exports.APIContracts = void 0;
|
|
27
|
+
const api_1 = __importStar(require("./api"));
|
|
28
|
+
Object.defineProperty(exports, "cloudApi", { enumerable: true, get: function () { return api_1.cloudApi; } });
|
|
29
|
+
Object.defineProperty(exports, "CloudApi", { enumerable: true, get: function () { return api_1.CloudApi; } });
|
|
31
30
|
const Api_1 = require("./generated/Api");
|
|
32
31
|
Object.defineProperty(exports, "Api", { enumerable: true, get: function () { return Api_1.Api; } });
|
|
33
32
|
const APIContracts = __importStar(require("./generated/data-contracts"));
|
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
import { ManagedWorkerRegion } from '../../rest/generated/cloud/data-contracts';
|
|
2
|
+
import { z } from 'zod';
|
|
3
|
+
export declare const SharedCPUComputeSchema: z.ZodObject<z.objectUtil.extendShape<{
|
|
4
|
+
pool: z.ZodOptional<z.ZodDefault<z.ZodString>>;
|
|
5
|
+
numReplicas: z.ZodDefault<z.ZodNumber>;
|
|
6
|
+
regions: z.ZodOptional<z.ZodArray<z.ZodNativeEnum<typeof ManagedWorkerRegion>, "many">>;
|
|
7
|
+
cpus: z.ZodNumber;
|
|
8
|
+
}, {
|
|
9
|
+
cpuKind: z.ZodLiteral<"shared">;
|
|
10
|
+
memoryMb: z.ZodEffects<z.ZodNumber, number, number>;
|
|
11
|
+
}>, "strip", z.ZodTypeAny, {
|
|
12
|
+
numReplicas: number;
|
|
13
|
+
cpus: number;
|
|
14
|
+
cpuKind: "shared";
|
|
15
|
+
memoryMb: number;
|
|
16
|
+
pool?: string | undefined;
|
|
17
|
+
regions?: ManagedWorkerRegion[] | undefined;
|
|
18
|
+
}, {
|
|
19
|
+
cpus: number;
|
|
20
|
+
cpuKind: "shared";
|
|
21
|
+
memoryMb: number;
|
|
22
|
+
pool?: string | undefined;
|
|
23
|
+
numReplicas?: number | undefined;
|
|
24
|
+
regions?: ManagedWorkerRegion[] | undefined;
|
|
25
|
+
}>;
|
|
26
|
+
export type SharedCPUCompute = z.infer<typeof SharedCPUComputeSchema>;
|
|
27
|
+
export declare const PerformanceCPUComputeSchema: z.ZodObject<z.objectUtil.extendShape<{
|
|
28
|
+
pool: z.ZodOptional<z.ZodDefault<z.ZodString>>;
|
|
29
|
+
numReplicas: z.ZodDefault<z.ZodNumber>;
|
|
30
|
+
regions: z.ZodOptional<z.ZodArray<z.ZodNativeEnum<typeof ManagedWorkerRegion>, "many">>;
|
|
31
|
+
cpus: z.ZodNumber;
|
|
32
|
+
}, {
|
|
33
|
+
cpuKind: z.ZodLiteral<"performance">;
|
|
34
|
+
memoryMb: z.ZodEffects<z.ZodNumber, number, number>;
|
|
35
|
+
}>, "strip", z.ZodTypeAny, {
|
|
36
|
+
numReplicas: number;
|
|
37
|
+
cpus: number;
|
|
38
|
+
cpuKind: "performance";
|
|
39
|
+
memoryMb: number;
|
|
40
|
+
pool?: string | undefined;
|
|
41
|
+
regions?: ManagedWorkerRegion[] | undefined;
|
|
42
|
+
}, {
|
|
43
|
+
cpus: number;
|
|
44
|
+
cpuKind: "performance";
|
|
45
|
+
memoryMb: number;
|
|
46
|
+
pool?: string | undefined;
|
|
47
|
+
numReplicas?: number | undefined;
|
|
48
|
+
regions?: ManagedWorkerRegion[] | undefined;
|
|
49
|
+
}>;
|
|
50
|
+
export type PerformanceCPUCompute = z.infer<typeof PerformanceCPUComputeSchema>;
|
|
51
|
+
export declare const GPUComputeSchema: z.ZodObject<z.objectUtil.extendShape<{
|
|
52
|
+
pool: z.ZodOptional<z.ZodDefault<z.ZodString>>;
|
|
53
|
+
numReplicas: z.ZodDefault<z.ZodNumber>;
|
|
54
|
+
regions: z.ZodOptional<z.ZodArray<z.ZodNativeEnum<typeof ManagedWorkerRegion>, "many">>;
|
|
55
|
+
cpus: z.ZodNumber;
|
|
56
|
+
}, {
|
|
57
|
+
cpuKind: z.ZodLiteral<"shared">;
|
|
58
|
+
gpuKind: z.ZodEnum<["a10", "l40s", "a100-40gb", "a100-80gb"]>;
|
|
59
|
+
regions: z.ZodOptional<z.ZodEffects<z.ZodArray<z.ZodNativeEnum<typeof ManagedWorkerRegion>, "many">, ManagedWorkerRegion[], ManagedWorkerRegion[]>>;
|
|
60
|
+
memoryMb: z.ZodEffects<z.ZodNumber, number, number>;
|
|
61
|
+
}>, "strip", z.ZodTypeAny, {
|
|
62
|
+
numReplicas: number;
|
|
63
|
+
cpus: number;
|
|
64
|
+
cpuKind: "shared";
|
|
65
|
+
memoryMb: number;
|
|
66
|
+
gpuKind: "a10" | "l40s" | "a100-40gb" | "a100-80gb";
|
|
67
|
+
pool?: string | undefined;
|
|
68
|
+
regions?: ManagedWorkerRegion[] | undefined;
|
|
69
|
+
}, {
|
|
70
|
+
cpus: number;
|
|
71
|
+
cpuKind: "shared";
|
|
72
|
+
memoryMb: number;
|
|
73
|
+
gpuKind: "a10" | "l40s" | "a100-40gb" | "a100-80gb";
|
|
74
|
+
pool?: string | undefined;
|
|
75
|
+
numReplicas?: number | undefined;
|
|
76
|
+
regions?: ManagedWorkerRegion[] | undefined;
|
|
77
|
+
}>;
|
|
78
|
+
export type GPUCompute = z.infer<typeof GPUComputeSchema>;
|
|
79
|
+
export declare const ComputeSchema: z.ZodUnion<[z.ZodObject<z.objectUtil.extendShape<{
|
|
80
|
+
pool: z.ZodOptional<z.ZodDefault<z.ZodString>>;
|
|
81
|
+
numReplicas: z.ZodDefault<z.ZodNumber>;
|
|
82
|
+
regions: z.ZodOptional<z.ZodArray<z.ZodNativeEnum<typeof ManagedWorkerRegion>, "many">>;
|
|
83
|
+
cpus: z.ZodNumber;
|
|
84
|
+
}, {
|
|
85
|
+
cpuKind: z.ZodLiteral<"shared">;
|
|
86
|
+
memoryMb: z.ZodEffects<z.ZodNumber, number, number>;
|
|
87
|
+
}>, "strip", z.ZodTypeAny, {
|
|
88
|
+
numReplicas: number;
|
|
89
|
+
cpus: number;
|
|
90
|
+
cpuKind: "shared";
|
|
91
|
+
memoryMb: number;
|
|
92
|
+
pool?: string | undefined;
|
|
93
|
+
regions?: ManagedWorkerRegion[] | undefined;
|
|
94
|
+
}, {
|
|
95
|
+
cpus: number;
|
|
96
|
+
cpuKind: "shared";
|
|
97
|
+
memoryMb: number;
|
|
98
|
+
pool?: string | undefined;
|
|
99
|
+
numReplicas?: number | undefined;
|
|
100
|
+
regions?: ManagedWorkerRegion[] | undefined;
|
|
101
|
+
}>, z.ZodObject<z.objectUtil.extendShape<{
|
|
102
|
+
pool: z.ZodOptional<z.ZodDefault<z.ZodString>>;
|
|
103
|
+
numReplicas: z.ZodDefault<z.ZodNumber>;
|
|
104
|
+
regions: z.ZodOptional<z.ZodArray<z.ZodNativeEnum<typeof ManagedWorkerRegion>, "many">>;
|
|
105
|
+
cpus: z.ZodNumber;
|
|
106
|
+
}, {
|
|
107
|
+
cpuKind: z.ZodLiteral<"performance">;
|
|
108
|
+
memoryMb: z.ZodEffects<z.ZodNumber, number, number>;
|
|
109
|
+
}>, "strip", z.ZodTypeAny, {
|
|
110
|
+
numReplicas: number;
|
|
111
|
+
cpus: number;
|
|
112
|
+
cpuKind: "performance";
|
|
113
|
+
memoryMb: number;
|
|
114
|
+
pool?: string | undefined;
|
|
115
|
+
regions?: ManagedWorkerRegion[] | undefined;
|
|
116
|
+
}, {
|
|
117
|
+
cpus: number;
|
|
118
|
+
cpuKind: "performance";
|
|
119
|
+
memoryMb: number;
|
|
120
|
+
pool?: string | undefined;
|
|
121
|
+
numReplicas?: number | undefined;
|
|
122
|
+
regions?: ManagedWorkerRegion[] | undefined;
|
|
123
|
+
}>, z.ZodObject<z.objectUtil.extendShape<{
|
|
124
|
+
pool: z.ZodOptional<z.ZodDefault<z.ZodString>>;
|
|
125
|
+
numReplicas: z.ZodDefault<z.ZodNumber>;
|
|
126
|
+
regions: z.ZodOptional<z.ZodArray<z.ZodNativeEnum<typeof ManagedWorkerRegion>, "many">>;
|
|
127
|
+
cpus: z.ZodNumber;
|
|
128
|
+
}, {
|
|
129
|
+
cpuKind: z.ZodLiteral<"shared">;
|
|
130
|
+
gpuKind: z.ZodEnum<["a10", "l40s", "a100-40gb", "a100-80gb"]>;
|
|
131
|
+
regions: z.ZodOptional<z.ZodEffects<z.ZodArray<z.ZodNativeEnum<typeof ManagedWorkerRegion>, "many">, ManagedWorkerRegion[], ManagedWorkerRegion[]>>;
|
|
132
|
+
memoryMb: z.ZodEffects<z.ZodNumber, number, number>;
|
|
133
|
+
}>, "strip", z.ZodTypeAny, {
|
|
134
|
+
numReplicas: number;
|
|
135
|
+
cpus: number;
|
|
136
|
+
cpuKind: "shared";
|
|
137
|
+
memoryMb: number;
|
|
138
|
+
gpuKind: "a10" | "l40s" | "a100-40gb" | "a100-80gb";
|
|
139
|
+
pool?: string | undefined;
|
|
140
|
+
regions?: ManagedWorkerRegion[] | undefined;
|
|
141
|
+
}, {
|
|
142
|
+
cpus: number;
|
|
143
|
+
cpuKind: "shared";
|
|
144
|
+
memoryMb: number;
|
|
145
|
+
gpuKind: "a10" | "l40s" | "a100-40gb" | "a100-80gb";
|
|
146
|
+
pool?: string | undefined;
|
|
147
|
+
numReplicas?: number | undefined;
|
|
148
|
+
regions?: ManagedWorkerRegion[] | undefined;
|
|
149
|
+
}>]>;
|
|
150
|
+
export declare const computeHash: (compute: z.infer<typeof ComputeSchema>) => string;
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.computeHash = exports.ComputeSchema = exports.GPUComputeSchema = exports.PerformanceCPUComputeSchema = exports.SharedCPUComputeSchema = void 0;
|
|
4
|
+
const data_contracts_1 = require("../../rest/generated/cloud/data-contracts");
|
|
5
|
+
const crypto_1 = require("crypto");
|
|
6
|
+
const zod_1 = require("zod");
|
|
7
|
+
const BaseComputeSchema = zod_1.z.object({
|
|
8
|
+
pool: zod_1.z.string().default('default').optional().describe('The name of the compute pool to use'),
|
|
9
|
+
numReplicas: zod_1.z
|
|
10
|
+
.number()
|
|
11
|
+
.int()
|
|
12
|
+
.min(0, { message: 'Must be at least 0' })
|
|
13
|
+
.max(1000, { message: 'Must be at most 1000' })
|
|
14
|
+
.default(1)
|
|
15
|
+
.describe('The number of replicas'),
|
|
16
|
+
regions: zod_1.z
|
|
17
|
+
.array(zod_1.z.nativeEnum(data_contracts_1.ManagedWorkerRegion))
|
|
18
|
+
.optional()
|
|
19
|
+
.describe('The regions to deploy the worker to'),
|
|
20
|
+
cpus: zod_1.z
|
|
21
|
+
.number()
|
|
22
|
+
.int()
|
|
23
|
+
.min(1, { message: 'Must be at least 1' })
|
|
24
|
+
.max(64, { message: 'Must be at most 64' })
|
|
25
|
+
.describe('The number of CPUs to use for the worker'),
|
|
26
|
+
});
|
|
27
|
+
exports.SharedCPUComputeSchema = BaseComputeSchema.extend({
|
|
28
|
+
cpuKind: zod_1.z.literal('shared'),
|
|
29
|
+
memoryMb: zod_1.z
|
|
30
|
+
.number()
|
|
31
|
+
.int()
|
|
32
|
+
.min(256, { message: 'Must be at least 256 MB' })
|
|
33
|
+
.max(65536, { message: 'Must be at most 65536 MB' })
|
|
34
|
+
.refine((val) => val % 256 === 0, { message: 'Must be divisible by 256 MB' })
|
|
35
|
+
.describe('The amount of memory in MB to use for the worker'),
|
|
36
|
+
});
|
|
37
|
+
exports.PerformanceCPUComputeSchema = BaseComputeSchema.extend({
|
|
38
|
+
cpuKind: zod_1.z.literal('performance'),
|
|
39
|
+
memoryMb: zod_1.z
|
|
40
|
+
.number()
|
|
41
|
+
.int()
|
|
42
|
+
.min(2048, { message: 'Must be at least 1024 MB' })
|
|
43
|
+
.max(65536, { message: 'Must be at most 65536 MB' })
|
|
44
|
+
.refine((val) => val % 256 === 0, { message: 'Must be divisible by 256 MB' })
|
|
45
|
+
.describe('The amount of memory in MB to use for the worker'),
|
|
46
|
+
});
|
|
47
|
+
// eslint-disable-next-line no-shadow
|
|
48
|
+
const AllowedGPUManagedWorkerRegions = [data_contracts_1.ManagedWorkerRegion.Ord];
|
|
49
|
+
exports.GPUComputeSchema = BaseComputeSchema.extend({
|
|
50
|
+
cpuKind: zod_1.z.literal('shared'),
|
|
51
|
+
gpuKind: zod_1.z.enum(['a10', 'l40s', 'a100-40gb', 'a100-80gb']),
|
|
52
|
+
regions: zod_1.z
|
|
53
|
+
.array(zod_1.z.nativeEnum(data_contracts_1.ManagedWorkerRegion))
|
|
54
|
+
.refine((val) => val.every((region) => AllowedGPUManagedWorkerRegions.includes(region)), {
|
|
55
|
+
message: 'Invalid GPU region',
|
|
56
|
+
})
|
|
57
|
+
.optional()
|
|
58
|
+
.describe('The regions to deploy the worker to'),
|
|
59
|
+
memoryMb: zod_1.z
|
|
60
|
+
.number()
|
|
61
|
+
.int()
|
|
62
|
+
.min(2048, { message: 'Must be at least 1024 MB' })
|
|
63
|
+
.max(65536, { message: 'Must be at most 65536 MB' })
|
|
64
|
+
.refine((val) => val % 256 === 0, { message: 'Must be divisible by 256 MB' })
|
|
65
|
+
.describe('The amount of memory in MB to use for the worker'),
|
|
66
|
+
});
|
|
67
|
+
exports.ComputeSchema = zod_1.z.union([
|
|
68
|
+
exports.SharedCPUComputeSchema,
|
|
69
|
+
exports.PerformanceCPUComputeSchema,
|
|
70
|
+
exports.GPUComputeSchema,
|
|
71
|
+
]);
|
|
72
|
+
const computeHash = (compute) => {
|
|
73
|
+
const str = JSON.stringify(compute);
|
|
74
|
+
return (0, crypto_1.createHash)('sha256').update(str).digest('hex');
|
|
75
|
+
};
|
|
76
|
+
exports.computeHash = computeHash;
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import { HatchetClient } from '../../hatchet-client';
|
|
2
|
+
import { CreateManagedWorkerRuntimeConfigRequest } from '../../rest/generated/cloud/data-contracts';
|
|
3
|
+
import { Logger } from '../../../util/logger';
|
|
4
|
+
import { ActionRegistry } from '../worker';
|
|
5
|
+
export declare class ManagedCompute {
|
|
6
|
+
actions: ActionRegistry;
|
|
7
|
+
client: HatchetClient;
|
|
8
|
+
maxRuns: number;
|
|
9
|
+
configs: CreateManagedWorkerRuntimeConfigRequest[];
|
|
10
|
+
cloudRegisterId: string | undefined;
|
|
11
|
+
logger: Logger;
|
|
12
|
+
constructor(actions: ActionRegistry, client: HatchetClient, maxRuns?: number);
|
|
13
|
+
getComputeConfigs(actions: ActionRegistry): CreateManagedWorkerRuntimeConfigRequest[];
|
|
14
|
+
cloudRegister(): Promise<void>;
|
|
15
|
+
}
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
3
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
4
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
5
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
6
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
7
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
8
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
|
+
});
|
|
10
|
+
};
|
|
11
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
|
+
exports.ManagedCompute = void 0;
|
|
13
|
+
const logger_1 = require("../../../util/logger");
|
|
14
|
+
const compute_config_1 = require("./compute-config");
|
|
15
|
+
class ManagedCompute {
|
|
16
|
+
constructor(actions, client, maxRuns = 1) {
|
|
17
|
+
this.actions = actions;
|
|
18
|
+
this.client = client;
|
|
19
|
+
this.maxRuns = maxRuns;
|
|
20
|
+
this.configs = this.getComputeConfigs(this.actions);
|
|
21
|
+
this.cloudRegisterId = this.client.config.cloud_register_id;
|
|
22
|
+
this.logger = new logger_1.Logger(`Compute`, this.client.config.log_level);
|
|
23
|
+
if (this.configs.length === 0) {
|
|
24
|
+
this.logger.debug('No compute configs found, skipping cloud registration and running all actions locally.');
|
|
25
|
+
return;
|
|
26
|
+
}
|
|
27
|
+
if (!this.cloudRegisterId) {
|
|
28
|
+
this.logger.warn('Managed cloud compute plan:');
|
|
29
|
+
this.configs.forEach((compute) => {
|
|
30
|
+
var _a;
|
|
31
|
+
this.logger.warn(' ----------------------------');
|
|
32
|
+
this.logger.warn(` actions: ${(_a = compute.actions) === null || _a === void 0 ? void 0 : _a.join(', ')}`);
|
|
33
|
+
this.logger.warn(` num replicas: ${compute.numReplicas}`);
|
|
34
|
+
this.logger.warn(` cpu kind: ${compute.cpuKind}`);
|
|
35
|
+
this.logger.warn(` cpus: ${compute.cpus}`);
|
|
36
|
+
this.logger.warn(` memory mb: ${compute.memoryMb}`);
|
|
37
|
+
// this.logger.warn(` regions: ${compute.regions?.join(', ')}`);
|
|
38
|
+
});
|
|
39
|
+
this.logger.warn('NOTICE: local mode detected, skipping cloud registration and running all actions locally.');
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
getComputeConfigs(actions) {
|
|
43
|
+
/**
|
|
44
|
+
* Builds a map of compute hashes to compute configs and lists of actions that correspond to each compute hash.
|
|
45
|
+
*/
|
|
46
|
+
const computeMap = {};
|
|
47
|
+
try {
|
|
48
|
+
Object.entries(actions).forEach(([action, { compute }]) => {
|
|
49
|
+
if (!compute) {
|
|
50
|
+
return;
|
|
51
|
+
}
|
|
52
|
+
const key = (0, compute_config_1.computeHash)(compute);
|
|
53
|
+
if (!computeMap[key]) {
|
|
54
|
+
computeMap[key] = {
|
|
55
|
+
actions: [],
|
|
56
|
+
numReplicas: compute.numReplicas,
|
|
57
|
+
cpuKind: compute.cpuKind,
|
|
58
|
+
cpus: compute.cpus,
|
|
59
|
+
memoryMb: compute.memoryMb,
|
|
60
|
+
regions: compute.regions,
|
|
61
|
+
slots: this.maxRuns,
|
|
62
|
+
};
|
|
63
|
+
}
|
|
64
|
+
computeMap[key].actions.push(action);
|
|
65
|
+
});
|
|
66
|
+
return Object.values(computeMap);
|
|
67
|
+
}
|
|
68
|
+
catch (e) {
|
|
69
|
+
this.logger.error(`Error getting compute configs: ${e}`);
|
|
70
|
+
return [];
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
cloudRegister() {
|
|
74
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
75
|
+
/**
|
|
76
|
+
* Registers the compute plan with the cloud if the environment variable is set.
|
|
77
|
+
* Exits the process upon completion.
|
|
78
|
+
*/
|
|
79
|
+
if (this.cloudRegisterId) {
|
|
80
|
+
this.logger.info(`Registering cloud compute plan with ID: ${this.cloudRegisterId}`);
|
|
81
|
+
try {
|
|
82
|
+
if (this.configs.length === 0) {
|
|
83
|
+
this.logger.warn('No actions to register, skipping cloud registration.');
|
|
84
|
+
process.exit(0);
|
|
85
|
+
}
|
|
86
|
+
const req = {
|
|
87
|
+
runtimeConfigs: this.configs,
|
|
88
|
+
};
|
|
89
|
+
yield this.client.cloudApi.infraAsCodeCreate(this.cloudRegisterId, req);
|
|
90
|
+
process.exit(0);
|
|
91
|
+
}
|
|
92
|
+
catch (e) {
|
|
93
|
+
this.logger.error(`ERROR: ${e}`);
|
|
94
|
+
process.exit(1);
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
});
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
exports.ManagedCompute = ManagedCompute;
|
|
@@ -6,9 +6,16 @@ import { Workflow } from '../../workflow';
|
|
|
6
6
|
import { Logger } from '../../util/logger';
|
|
7
7
|
import { WebhookHandler } from './handler';
|
|
8
8
|
import { WebhookWorkerCreateRequest } from '../rest/generated/data-contracts';
|
|
9
|
+
import { z } from 'zod';
|
|
9
10
|
import { Context, StepRunFunction } from '../../step';
|
|
10
11
|
import { WorkerLabels } from '../dispatcher/dispatcher-client';
|
|
11
|
-
|
|
12
|
+
import { ComputeSchema } from './compute/compute-config';
|
|
13
|
+
type ActionFunction = StepRunFunction<any, any> | ((ctx: any) => string | Promise<string>);
|
|
14
|
+
type ActionRegistryRecord = {
|
|
15
|
+
func: ActionFunction;
|
|
16
|
+
compute?: z.infer<typeof ComputeSchema>;
|
|
17
|
+
};
|
|
18
|
+
export type ActionRegistry = Record<Action['actionId'], ActionRegistryRecord>;
|
|
12
19
|
export interface WorkerOpts {
|
|
13
20
|
name: string;
|
|
14
21
|
handleKill?: boolean;
|
|
@@ -44,7 +51,7 @@ export declare class Worker {
|
|
|
44
51
|
*/
|
|
45
52
|
register_workflow(initWorkflow: Workflow): Promise<void>;
|
|
46
53
|
registerWorkflow(initWorkflow: Workflow): Promise<void>;
|
|
47
|
-
registerAction<T, K>(actionId: string, action: StepRunFunction<T, K>): void;
|
|
54
|
+
registerAction<T, K>(actionId: string, action: StepRunFunction<T, K>, compute?: z.infer<typeof ComputeSchema>): void;
|
|
48
55
|
handleStartStepRun(action: Action): Promise<void>;
|
|
49
56
|
handleStartGroupKeyRun(action: Action): Promise<void>;
|
|
50
57
|
getStepActionEvent(action: Action, eventType: StepActionEventType, payload?: any): StepActionEvent;
|
|
@@ -56,3 +63,4 @@ export declare class Worker {
|
|
|
56
63
|
handleAction(action: Action): Promise<void>;
|
|
57
64
|
upsertLabels(labels: WorkerLabels): Promise<WorkerLabels>;
|
|
58
65
|
}
|
|
66
|
+
export {};
|
package/clients/worker/worker.js
CHANGED
|
@@ -27,6 +27,7 @@ const workflows_1 = require("../../protoc/workflows");
|
|
|
27
27
|
const logger_1 = require("../../util/logger");
|
|
28
28
|
const handler_1 = require("./handler");
|
|
29
29
|
const step_1 = require("../../step");
|
|
30
|
+
const managed_compute_1 = require("./compute/managed-compute");
|
|
30
31
|
class Worker {
|
|
31
32
|
constructor(client, options) {
|
|
32
33
|
this.workflow_registry = [];
|
|
@@ -48,18 +49,33 @@ class Worker {
|
|
|
48
49
|
registerActions(workflow) {
|
|
49
50
|
var _a;
|
|
50
51
|
const newActions = workflow.steps.reduce((acc, step) => {
|
|
51
|
-
|
|
52
|
+
// Only register actions that are in the runnable_actions list
|
|
53
|
+
if (!this.client.config.runnable_actions ||
|
|
54
|
+
this.client.config.runnable_actions.includes(`${workflow.id}:${step.name}`)) {
|
|
55
|
+
acc[`${workflow.id}:${step.name}`] = {
|
|
56
|
+
func: step.run,
|
|
57
|
+
compute: step.compute,
|
|
58
|
+
};
|
|
59
|
+
}
|
|
52
60
|
return acc;
|
|
53
61
|
}, {});
|
|
54
62
|
const onFailureAction = workflow.onFailure
|
|
55
63
|
? {
|
|
56
|
-
[`${workflow.id}-on-failure:${workflow.onFailure.name}`]:
|
|
64
|
+
[`${workflow.id}-on-failure:${workflow.onFailure.name}`]: {
|
|
65
|
+
func: workflow.onFailure.run,
|
|
66
|
+
compute: workflow.onFailure.compute,
|
|
67
|
+
},
|
|
68
|
+
}
|
|
69
|
+
: {};
|
|
70
|
+
const concurrencyAction = ((_a = workflow.concurrency) === null || _a === void 0 ? void 0 : _a.name) && workflow.concurrency.key
|
|
71
|
+
? {
|
|
72
|
+
[`${workflow.id}:${workflow.concurrency.name}`]: {
|
|
73
|
+
func: workflow.concurrency.key,
|
|
74
|
+
compute: undefined,
|
|
75
|
+
},
|
|
57
76
|
}
|
|
58
77
|
: {};
|
|
59
|
-
this.action_registry = Object.assign(Object.assign(Object.assign({}, this.action_registry), newActions), onFailureAction);
|
|
60
|
-
this.action_registry =
|
|
61
|
-
((_a = workflow.concurrency) === null || _a === void 0 ? void 0 : _a.name) && workflow.concurrency.key
|
|
62
|
-
? Object.assign(Object.assign({}, this.action_registry), { [`${workflow.id}:${workflow.concurrency.name}`]: workflow.concurrency.key }) : Object.assign({}, this.action_registry);
|
|
78
|
+
this.action_registry = Object.assign(Object.assign(Object.assign(Object.assign({}, this.action_registry), newActions), onFailureAction), concurrencyAction);
|
|
63
79
|
}
|
|
64
80
|
getHandler(workflows) {
|
|
65
81
|
for (const workflow of workflows) {
|
|
@@ -162,8 +178,11 @@ class Worker {
|
|
|
162
178
|
this.registerActions(workflow);
|
|
163
179
|
});
|
|
164
180
|
}
|
|
165
|
-
registerAction(actionId, action) {
|
|
166
|
-
this.action_registry[actionId] =
|
|
181
|
+
registerAction(actionId, action, compute) {
|
|
182
|
+
this.action_registry[actionId] = {
|
|
183
|
+
func: action,
|
|
184
|
+
compute,
|
|
185
|
+
};
|
|
167
186
|
}
|
|
168
187
|
handleStartStepRun(action) {
|
|
169
188
|
return __awaiter(this, void 0, void 0, function* () {
|
|
@@ -177,7 +196,7 @@ class Worker {
|
|
|
177
196
|
return;
|
|
178
197
|
}
|
|
179
198
|
const run = () => __awaiter(this, void 0, void 0, function* () {
|
|
180
|
-
return step(context);
|
|
199
|
+
return step.func(context);
|
|
181
200
|
});
|
|
182
201
|
const success = (result) => __awaiter(this, void 0, void 0, function* () {
|
|
183
202
|
this.logger.info(`Step run ${action.stepRunId} succeeded`);
|
|
@@ -262,7 +281,7 @@ class Worker {
|
|
|
262
281
|
return;
|
|
263
282
|
}
|
|
264
283
|
const run = () => __awaiter(this, void 0, void 0, function* () {
|
|
265
|
-
return step(context);
|
|
284
|
+
return step.func(context);
|
|
266
285
|
});
|
|
267
286
|
const success = (result) => {
|
|
268
287
|
this.logger.info(`Step run ${action.stepRunId} succeeded`);
|
|
@@ -391,6 +410,8 @@ class Worker {
|
|
|
391
410
|
var _a, e_1, _b, _c;
|
|
392
411
|
// ensure all workflows are registered
|
|
393
412
|
yield Promise.all(this.registeredWorkflowPromises);
|
|
413
|
+
const managedCompute = new managed_compute_1.ManagedCompute(this.action_registry, this.client, this.maxRuns);
|
|
414
|
+
yield managedCompute.cloudRegister();
|
|
394
415
|
try {
|
|
395
416
|
this.listener = yield this.client.dispatcher.getActionListener({
|
|
396
417
|
workerName: this.name,
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
3
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
4
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
5
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
6
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
7
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
8
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
|
+
});
|
|
10
|
+
};
|
|
11
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
12
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
13
|
+
};
|
|
14
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
15
|
+
const data_contracts_1 = require("../clients/rest/generated/cloud/data-contracts");
|
|
16
|
+
const sdk_1 = __importDefault(require("../sdk"));
|
|
17
|
+
const hatchet = sdk_1.default.init();
|
|
18
|
+
const oneCpuWorkerConfig = {
|
|
19
|
+
cpuKind: 'shared',
|
|
20
|
+
memoryMb: 1024,
|
|
21
|
+
numReplicas: 1,
|
|
22
|
+
cpus: 1,
|
|
23
|
+
regions: [data_contracts_1.ManagedWorkerRegion.Ewr],
|
|
24
|
+
};
|
|
25
|
+
const twoCpuWorkerConfig = {
|
|
26
|
+
cpuKind: 'shared',
|
|
27
|
+
memoryMb: 1024,
|
|
28
|
+
numReplicas: 1,
|
|
29
|
+
cpus: 2,
|
|
30
|
+
regions: [data_contracts_1.ManagedWorkerRegion.Ewr],
|
|
31
|
+
};
|
|
32
|
+
const workflow = {
|
|
33
|
+
id: 'simple-workflow',
|
|
34
|
+
description: 'test',
|
|
35
|
+
on: {
|
|
36
|
+
event: 'user:create',
|
|
37
|
+
},
|
|
38
|
+
steps: [
|
|
39
|
+
{
|
|
40
|
+
name: 'step1',
|
|
41
|
+
compute: oneCpuWorkerConfig,
|
|
42
|
+
run: (ctx) => __awaiter(void 0, void 0, void 0, function* () {
|
|
43
|
+
console.log('executed step1!');
|
|
44
|
+
return { step1: 'step1 results!' };
|
|
45
|
+
}),
|
|
46
|
+
},
|
|
47
|
+
{
|
|
48
|
+
name: 'step2',
|
|
49
|
+
parents: ['step1'],
|
|
50
|
+
compute: twoCpuWorkerConfig,
|
|
51
|
+
run: (ctx) => {
|
|
52
|
+
console.log('executed step2 after step1 returned ', ctx.stepOutput('step1'));
|
|
53
|
+
return { step2: 'step2 results!' };
|
|
54
|
+
},
|
|
55
|
+
},
|
|
56
|
+
],
|
|
57
|
+
};
|
|
58
|
+
function main() {
|
|
59
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
60
|
+
const worker = yield hatchet.worker('managed-worker');
|
|
61
|
+
yield worker.registerWorkflow(workflow);
|
|
62
|
+
worker.start();
|
|
63
|
+
});
|
|
64
|
+
}
|
|
65
|
+
main();
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@hatchet-dev/typescript-sdk",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.16.0-alpha.2",
|
|
4
4
|
"description": "Background task orchestration & visibility for developers",
|
|
5
5
|
"types": "dist/index.d.ts",
|
|
6
6
|
"files": [
|
|
@@ -23,6 +23,7 @@
|
|
|
23
23
|
"test:unit:watch": "jest --testMatch='**/*.test.ts' --watch",
|
|
24
24
|
"generate": "pnpm run '/generate-.*/'",
|
|
25
25
|
"generate-api": "npx --yes swagger-cli bundle ./hatchet/api-contracts/openapi/openapi.yaml --outfile openapi.yaml --type yaml && npx swagger-typescript-api -p openapi.yaml -o src/clients/rest/generated -n hatchet.ts --modular --axios",
|
|
26
|
+
"generate-cloud-api": "npx --yes swagger-cli bundle ./hatchet-cloud/api-contracts/openapi/openapi.yaml --outfile openapi.yaml --type yaml && npx swagger-typescript-api -p openapi.yaml -o src/clients/rest/generated/cloud -n hatchet.ts --modular --axios",
|
|
26
27
|
"generate-protoc": "./generate-protoc.sh",
|
|
27
28
|
"lint:check": "npm run eslint:check && npm run prettier:check",
|
|
28
29
|
"lint:fix": "npm run eslint:fix && npm run prettier:fix",
|
|
@@ -38,6 +39,7 @@
|
|
|
38
39
|
"example:rate": "npm run exec -- ./src/examples/rate-limit/events.ts",
|
|
39
40
|
"worker:fanout": "npm run exec -- ./src/examples/fanout-worker.ts",
|
|
40
41
|
"worker:simple": "npm run exec -- ./src/examples/simple-worker.ts",
|
|
42
|
+
"worker:managed": "npm run exec -- ./src/examples/managed-worker.ts",
|
|
41
43
|
"worker:affinity": "npm run exec -- ./src/examples/affinity-workers.ts",
|
|
42
44
|
"worker:sticky": "npm run exec -- ./src/examples/sticky-worker.ts",
|
|
43
45
|
"worker:on-failure": "npm run exec -- ./src/examples/on-failure.ts",
|
|
@@ -58,6 +60,7 @@
|
|
|
58
60
|
"api": "npm run exec -- ./src/examples/api.ts",
|
|
59
61
|
"prepublish": "cp package.json dist/package.json;",
|
|
60
62
|
"publish:ci": "rm -rf ./dist && npm run tsc:build && npm run prepublish && cd dist && npm publish --access public --no-git-checks",
|
|
63
|
+
"publish:ci:alpha": "rm -rf ./dist && npm run tsc:build && npm run prepublish && cd dist && npm publish --access public --no-git-checks --tag alpha",
|
|
61
64
|
"generate-docs": "typedoc"
|
|
62
65
|
},
|
|
63
66
|
"keywords": [],
|
|
@@ -1,6 +1,15 @@
|
|
|
1
1
|
import { BinaryReader, BinaryWriter } from '@bufbuild/protobuf/wire';
|
|
2
2
|
import { type CallContext, type CallOptions } from 'nice-grpc-common';
|
|
3
3
|
export declare const protobufPackage = "";
|
|
4
|
+
export declare enum SDKS {
|
|
5
|
+
UNKNOWN = 0,
|
|
6
|
+
GO = 1,
|
|
7
|
+
PYTHON = 2,
|
|
8
|
+
TYPESCRIPT = 3,
|
|
9
|
+
UNRECOGNIZED = -1
|
|
10
|
+
}
|
|
11
|
+
export declare function sDKSFromJSON(object: any): SDKS;
|
|
12
|
+
export declare function sDKSToJSON(object: SDKS): string;
|
|
4
13
|
export declare enum ActionType {
|
|
5
14
|
START_STEP_RUN = 0,
|
|
6
15
|
CANCEL_STEP_RUN = 1,
|
|
@@ -59,6 +68,13 @@ export interface WorkerLabels {
|
|
|
59
68
|
strValue?: string | undefined;
|
|
60
69
|
intValue?: number | undefined;
|
|
61
70
|
}
|
|
71
|
+
export interface RuntimeInfo {
|
|
72
|
+
sdkVersion?: string | undefined;
|
|
73
|
+
language?: SDKS | undefined;
|
|
74
|
+
languageVersion?: string | undefined;
|
|
75
|
+
os?: string | undefined;
|
|
76
|
+
extra?: string | undefined;
|
|
77
|
+
}
|
|
62
78
|
export interface WorkerRegisterRequest {
|
|
63
79
|
/** the name of the worker */
|
|
64
80
|
workerName: string;
|
|
@@ -74,6 +90,8 @@ export interface WorkerRegisterRequest {
|
|
|
74
90
|
};
|
|
75
91
|
/** (optional) webhookId is the id of the webhook that the worker is associated with (if any) */
|
|
76
92
|
webhookId?: string | undefined;
|
|
93
|
+
/** (optional) information regarding the runtime environment of the worker */
|
|
94
|
+
runtimeInfo?: RuntimeInfo | undefined;
|
|
77
95
|
}
|
|
78
96
|
export interface WorkerRegisterRequest_LabelsEntry {
|
|
79
97
|
key: string;
|
|
@@ -274,6 +292,7 @@ export interface ReleaseSlotRequest {
|
|
|
274
292
|
export interface ReleaseSlotResponse {
|
|
275
293
|
}
|
|
276
294
|
export declare const WorkerLabels: MessageFns<WorkerLabels>;
|
|
295
|
+
export declare const RuntimeInfo: MessageFns<RuntimeInfo>;
|
|
277
296
|
export declare const WorkerRegisterRequest: MessageFns<WorkerRegisterRequest>;
|
|
278
297
|
export declare const WorkerRegisterRequest_LabelsEntry: MessageFns<WorkerRegisterRequest_LabelsEntry>;
|
|
279
298
|
export declare const WorkerRegisterResponse: MessageFns<WorkerRegisterResponse>;
|