@highstate/backend 0.9.14 → 0.9.16

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (144) hide show
  1. package/dist/chunk-RCB4AFGD.js +159 -0
  2. package/dist/chunk-RCB4AFGD.js.map +1 -0
  3. package/dist/chunk-WHALQHEZ.js +2017 -0
  4. package/dist/chunk-WHALQHEZ.js.map +1 -0
  5. package/dist/highstate.manifest.json +3 -3
  6. package/dist/index.js +6146 -2174
  7. package/dist/index.js.map +1 -1
  8. package/dist/library/worker/main.js +51 -159
  9. package/dist/library/worker/main.js.map +1 -1
  10. package/dist/shared/index.js +159 -43
  11. package/package.json +25 -7
  12. package/src/artifact/abstractions.ts +46 -0
  13. package/src/artifact/encryption.ts +69 -0
  14. package/src/artifact/factory.ts +36 -0
  15. package/src/artifact/index.ts +3 -0
  16. package/src/artifact/local.ts +142 -0
  17. package/src/business/api-key.ts +65 -0
  18. package/src/business/artifact.ts +288 -0
  19. package/src/business/backend-unlock.ts +10 -0
  20. package/src/business/index.ts +9 -0
  21. package/src/business/instance-lock.ts +124 -0
  22. package/src/business/instance-state.ts +292 -0
  23. package/src/business/operation.ts +251 -0
  24. package/src/business/project-unlock.ts +242 -0
  25. package/src/business/secret.ts +187 -0
  26. package/src/business/worker.ts +161 -0
  27. package/src/common/index.ts +2 -1
  28. package/src/common/performance.ts +44 -0
  29. package/src/common/tree.ts +33 -0
  30. package/src/common/utils.ts +40 -1
  31. package/src/config.ts +14 -10
  32. package/src/hotstate/abstractions.ts +48 -0
  33. package/src/hotstate/factory.ts +17 -0
  34. package/src/{secret → hotstate}/index.ts +1 -0
  35. package/src/hotstate/manager.ts +192 -0
  36. package/src/hotstate/memory.ts +100 -0
  37. package/src/hotstate/validation.ts +101 -0
  38. package/src/index.ts +2 -1
  39. package/src/library/abstractions.ts +10 -23
  40. package/src/library/factory.ts +2 -2
  41. package/src/library/local.ts +89 -102
  42. package/src/library/worker/evaluator.ts +14 -47
  43. package/src/library/worker/loader.lite.ts +41 -0
  44. package/src/library/worker/main.ts +14 -65
  45. package/src/library/worker/protocol.ts +8 -24
  46. package/src/lock/abstractions.ts +6 -0
  47. package/src/lock/factory.ts +15 -0
  48. package/src/{workspace → lock}/index.ts +1 -0
  49. package/src/lock/manager.ts +82 -0
  50. package/src/lock/memory.ts +19 -0
  51. package/src/orchestrator/manager.ts +131 -82
  52. package/src/orchestrator/operation-workset.ts +188 -77
  53. package/src/orchestrator/operation.ts +975 -284
  54. package/src/project/abstractions.ts +20 -7
  55. package/src/project/factory.ts +1 -1
  56. package/src/project/index.ts +0 -1
  57. package/src/project/local.ts +73 -17
  58. package/src/project/manager.ts +272 -131
  59. package/src/pubsub/abstractions.ts +13 -0
  60. package/src/pubsub/factory.ts +19 -0
  61. package/src/pubsub/index.ts +3 -0
  62. package/src/pubsub/local.ts +36 -0
  63. package/src/pubsub/manager.ts +100 -0
  64. package/src/pubsub/validation.ts +33 -0
  65. package/src/runner/abstractions.ts +135 -68
  66. package/src/runner/artifact-env.ts +160 -0
  67. package/src/runner/factory.ts +20 -5
  68. package/src/runner/force-abort.ts +117 -0
  69. package/src/runner/local.ts +281 -372
  70. package/src/{common → runner}/pulumi.ts +86 -37
  71. package/src/services.ts +193 -35
  72. package/src/shared/index.ts +3 -11
  73. package/src/shared/models/backend/index.ts +3 -0
  74. package/src/shared/models/backend/project.ts +63 -0
  75. package/src/shared/models/backend/unlock-method.ts +20 -0
  76. package/src/shared/models/base.ts +151 -0
  77. package/src/shared/models/errors.ts +5 -0
  78. package/src/shared/models/index.ts +4 -0
  79. package/src/shared/models/project/api-key.ts +62 -0
  80. package/src/shared/models/project/artifact.ts +113 -0
  81. package/src/shared/models/project/component.ts +45 -0
  82. package/src/shared/models/project/index.ts +14 -0
  83. package/src/shared/{project.ts → models/project/instance.ts} +12 -0
  84. package/src/shared/models/project/lock.ts +91 -0
  85. package/src/shared/{operation.ts → models/project/operation.ts} +43 -8
  86. package/src/shared/models/project/page.ts +57 -0
  87. package/src/shared/models/project/secret.ts +112 -0
  88. package/src/shared/models/project/service-account.ts +22 -0
  89. package/src/shared/models/project/state.ts +432 -0
  90. package/src/shared/models/project/terminal.ts +99 -0
  91. package/src/shared/models/project/trigger.ts +56 -0
  92. package/src/shared/models/project/unlock-method.ts +31 -0
  93. package/src/shared/models/project/worker.ts +105 -0
  94. package/src/shared/resolvers/graph-resolver.ts +74 -13
  95. package/src/shared/resolvers/index.ts +5 -0
  96. package/src/shared/resolvers/input-hash.ts +53 -15
  97. package/src/shared/resolvers/input.ts +1 -9
  98. package/src/shared/resolvers/registry.ts +7 -2
  99. package/src/shared/resolvers/state.ts +12 -0
  100. package/src/shared/resolvers/validation.ts +61 -20
  101. package/src/shared/{async-batcher.ts → utils/async-batcher.ts} +13 -1
  102. package/src/shared/utils/hash.ts +6 -0
  103. package/src/shared/utils/index.ts +3 -0
  104. package/src/shared/utils/promise-tracker.ts +23 -0
  105. package/src/state/abstractions.ts +330 -101
  106. package/src/state/encryption.ts +59 -0
  107. package/src/state/factory.ts +3 -5
  108. package/src/state/index.ts +3 -0
  109. package/src/state/keyring.ts +22 -0
  110. package/src/state/local/backend.ts +299 -0
  111. package/src/state/local/collection.ts +342 -0
  112. package/src/state/local/index.ts +2 -0
  113. package/src/state/manager.ts +804 -18
  114. package/src/state/repository/index.ts +2 -0
  115. package/src/state/repository/repository.index.ts +193 -0
  116. package/src/state/repository/repository.ts +458 -0
  117. package/src/terminal/{shared.ts → abstractions.ts} +3 -3
  118. package/src/terminal/docker.ts +18 -14
  119. package/src/terminal/factory.ts +3 -3
  120. package/src/terminal/index.ts +1 -1
  121. package/src/terminal/manager.ts +134 -80
  122. package/src/terminal/run.sh.ts +22 -10
  123. package/src/worker/abstractions.ts +42 -0
  124. package/src/worker/docker.ts +83 -0
  125. package/src/worker/factory.ts +20 -0
  126. package/src/worker/index.ts +3 -0
  127. package/src/worker/manager.ts +139 -0
  128. package/dist/chunk-C2TJAQAD.js +0 -937
  129. package/dist/chunk-C2TJAQAD.js.map +0 -1
  130. package/dist/chunk-WXDYCRTT.js +0 -234
  131. package/dist/chunk-WXDYCRTT.js.map +0 -1
  132. package/src/library/worker/loader.ts +0 -114
  133. package/src/preferences/shared.ts +0 -1
  134. package/src/project/lock.ts +0 -39
  135. package/src/secret/abstractions.ts +0 -59
  136. package/src/secret/factory.ts +0 -22
  137. package/src/secret/local.ts +0 -152
  138. package/src/shared/state.ts +0 -270
  139. package/src/shared/terminal.ts +0 -13
  140. package/src/state/local.ts +0 -612
  141. package/src/workspace/abstractions.ts +0 -41
  142. package/src/workspace/factory.ts +0 -14
  143. package/src/workspace/local.ts +0 -54
  144. /package/src/shared/{library.ts → models/backend/library.ts} +0 -0
@@ -1,152 +0,0 @@
1
- import type { SecretBackend } from "./abstractions"
2
- import type { Logger } from "pino"
3
- import { mapKeys, mapValues, pickBy } from "remeda"
4
- import { z } from "zod"
5
- import { LocalPulumiHost, resolveMainLocalProject, stringToValue, valueToString } from "../common"
6
-
7
- export const localSecretBackendConfig = z.object({
8
- HIGHSTATE_BACKEND_SECRET_PROJECT_PATH: z.string().optional(),
9
- HIGHSTATE_BACKEND_SECRET_PROJECT_NAME: z.string().optional(),
10
- })
11
-
12
- /**
13
- * The backend for storing secrets in the local Pulumi project.
14
- * It is the simplest backend and should be used for development and single-user projects.
15
- */
16
- export class LocalSecretBackend implements SecretBackend {
17
- private constructor(
18
- private readonly projectPath: string,
19
- private readonly projectName: string,
20
- private readonly pulumiProjectHost: LocalPulumiHost,
21
- private readonly logger: Logger,
22
- ) {
23
- this.logger.debug({ msg: "initialized", projectPath, projectName })
24
- }
25
-
26
- isLocked(projectId: string): Promise<boolean> {
27
- return Promise.resolve(!this.pulumiProjectHost.hasPassword(projectId))
28
- }
29
-
30
- async unlock(projectId: string, password: string, signal?: AbortSignal): Promise<boolean> {
31
- this.pulumiProjectHost.setPassword(projectId, password)
32
-
33
- try {
34
- // try to run a command to check if the password is correct
35
- await this.pulumiProjectHost.runLocal(
36
- {
37
- projectId,
38
- pulumiProjectName: this.projectName,
39
- pulumiStackName: projectId,
40
- projectPath: this.projectPath,
41
- },
42
- async stack => {
43
- this.logger.debug({ projectId }, "checking password")
44
-
45
- await stack.info(true)
46
- },
47
- signal,
48
- )
49
-
50
- return true
51
- } catch (error) {
52
- if (error instanceof Error) {
53
- if (error.message.includes("incorrect passphrase")) {
54
- this.logger.debug({ projectId, error }, "incorrect passphrase")
55
-
56
- this.pulumiProjectHost.removePassword(projectId)
57
- return false
58
- }
59
- }
60
-
61
- throw error
62
- }
63
- }
64
-
65
- get(
66
- projectId: string,
67
- instanceId: string,
68
- signal?: AbortSignal,
69
- ): Promise<Record<string, unknown>> {
70
- return this.pulumiProjectHost.runLocal(
71
- {
72
- projectId,
73
- pulumiProjectName: this.projectName,
74
- pulumiStackName: projectId,
75
- projectPath: this.projectPath,
76
- },
77
- async stack => {
78
- this.logger.debug({ projectId, instanceId }, "getting secrets")
79
-
80
- const config = await stack.getAllConfig()
81
- signal?.throwIfAborted()
82
-
83
- const prefix = this.getPrefix(projectId, instanceId)
84
- const secrets = pickBy(config, (_, key) => key.startsWith(prefix))
85
- const trimmedSecrets = mapKeys(secrets, key => key.slice(prefix.length))
86
-
87
- return mapValues(trimmedSecrets, value => stringToValue(value.value))
88
- },
89
- signal,
90
- )
91
- }
92
-
93
- set(
94
- projectId: string,
95
- instanceId: string,
96
- values: Record<string, unknown>,
97
- signal?: AbortSignal,
98
- ): Promise<void> {
99
- return this.pulumiProjectHost.runLocal(
100
- {
101
- projectId,
102
- pulumiProjectName: this.projectName,
103
- pulumiStackName: projectId,
104
- projectPath: this.projectPath,
105
- },
106
- async stack => {
107
- this.logger.debug({ projectId, instanceId }, "setting secrets")
108
-
109
- const componentSecrets = mapValues(
110
- mapKeys(values, key => `${this.getPrefix(projectId, instanceId)}${key}`),
111
- value => ({
112
- value: valueToString(value),
113
- secret: true,
114
- }),
115
- )
116
-
117
- const config = await stack.getAllConfig()
118
- signal?.throwIfAborted()
119
-
120
- Object.assign(config, componentSecrets)
121
-
122
- await stack.setAllConfig(config)
123
- },
124
- signal,
125
- )
126
- }
127
-
128
- private getPrefix(projectId: string, instanceId: string) {
129
- // replace all semicolons with dashes since extra semicolons are not allowed in Pulumi config keys
130
- instanceId = instanceId.replace(/:/g, "_")
131
-
132
- return `${this.projectName}:${projectId}/${instanceId}/`
133
- }
134
-
135
- static async create(
136
- config: z.infer<typeof localSecretBackendConfig>,
137
- pulumiProjectHost: LocalPulumiHost,
138
- logger: Logger,
139
- ) {
140
- const [projectPath, projectName] = await resolveMainLocalProject(
141
- config.HIGHSTATE_BACKEND_SECRET_PROJECT_PATH,
142
- config.HIGHSTATE_BACKEND_SECRET_PROJECT_NAME,
143
- )
144
-
145
- return new LocalSecretBackend(
146
- projectPath,
147
- projectName,
148
- pulumiProjectHost,
149
- logger.child({ backend: "SecretBackend", service: "LocalSecretBackend" }),
150
- )
151
- }
152
- }
@@ -1,270 +0,0 @@
1
- import { omit } from "remeda"
2
- import { z } from "zod"
3
-
4
- export const instanceStatusSchema = z.enum([
5
- "not_created",
6
- "updating",
7
- "previewing",
8
- "destroying",
9
- "refreshing",
10
- "created",
11
- "error",
12
- "pending",
13
- "unknown",
14
- ])
15
-
16
- export const instanceStatusFieldValueSchema = z.union([
17
- z.string(),
18
- z.number(),
19
- z.boolean(),
20
- z.array(z.string()),
21
- ])
22
-
23
- export const instanceStatusFieldSchema = z.object({
24
- name: z.string(),
25
- value: instanceStatusFieldValueSchema.optional(),
26
- displayName: z.string().optional(),
27
- sensitive: z.boolean().optional(),
28
- url: z.string().optional(),
29
- complementaryTo: z.string().optional(),
30
- })
31
-
32
- export const instanceFileMetaSchema = z.object({
33
- name: z.string(),
34
- contentType: z.string(),
35
- size: z.number(),
36
- isBinary: z.boolean().optional(),
37
- })
38
-
39
- export const instanceFileSchema = z.object({
40
- meta: instanceFileMetaSchema,
41
- content: z.string(),
42
- })
43
-
44
- export const instancePageBlockSchema = z.union([
45
- z.object({
46
- type: z.literal("markdown"),
47
- content: z.string(),
48
- }),
49
- z.object({
50
- type: z.literal("qr"),
51
- content: z.string(),
52
- showContent: z.coerce.boolean(),
53
- language: z.string().optional(),
54
- }),
55
- z.object({
56
- type: z.literal("file"),
57
- fileMeta: instanceFileMetaSchema,
58
- }),
59
- ])
60
-
61
- export const instancePageMetaSchema = z.object({
62
- name: z.string(),
63
- title: z.string(),
64
- })
65
-
66
- export const instancePageSchema = z.object({
67
- ...instancePageMetaSchema.shape,
68
- content: z.array(instancePageBlockSchema),
69
- })
70
-
71
- export const instanceTerminalMetaSchema = z.object({
72
- name: z.string(),
73
- title: z.string(),
74
- description: z.string().optional(),
75
- })
76
-
77
- export const instanceTerminalFileSchema = z.object({
78
- content: z.string().optional(),
79
- isBinary: z.boolean().optional(),
80
- mode: z.number().optional(),
81
- })
82
-
83
- export const instanceTerminalSchema = z.object({
84
- ...instanceTerminalMetaSchema.shape,
85
- image: z.string(),
86
- command: z.string().array(),
87
- cwd: z.string().optional(),
88
- env: z.record(z.string()).optional(),
89
- files: z.record(instanceTerminalFileSchema).optional(),
90
- })
91
-
92
- export const instanceTriggerSpecSchema = z.union([
93
- z.object({
94
- type: z.literal("before-destroy"),
95
- }),
96
- z.object({
97
- type: z.literal("schedule"),
98
- schedule: z.string(),
99
- }),
100
- ])
101
-
102
- export const instanceTriggerSchema = z.object({
103
- name: z.string(),
104
- title: z.string(),
105
- description: z.string().optional(),
106
- spec: instanceTriggerSpecSchema,
107
- })
108
-
109
- export const instanceTriggerInvocationSchema = z.object({
110
- name: z.string(),
111
- })
112
-
113
- export const instanceStateSchema = z.object({
114
- id: z.string(),
115
- parentId: z.string().nullable().default(null),
116
- status: instanceStatusSchema,
117
- dependencyIds: z.array(z.string()).default(() => []),
118
-
119
- latestOperationId: z.string().nullable().default(null),
120
-
121
- currentResourceCount: z.number().nullable().default(null),
122
- totalResourceCount: z.number().nullable().default(null),
123
- inputHash: z.string().nullable().default(null),
124
- outputHash: z.string().nullable().default(null),
125
-
126
- error: z.string().nullable().default(null),
127
- evaluationError: z.string().nullable().default(null),
128
-
129
- statusFields: z.array(instanceStatusFieldSchema).default(() => []),
130
- files: z.array(instanceFileMetaSchema).default(() => []),
131
- pages: z.array(instancePageMetaSchema).default(() => []),
132
- terminals: z.array(instanceTerminalMetaSchema).default(() => []),
133
- triggers: z.array(instanceTriggerSchema).default(() => []),
134
- })
135
-
136
- export const instanceStateUpdateSchema = z
137
- .object({
138
- ...instanceStateSchema.shape,
139
-
140
- // secrets updated by the unit
141
- secrets: z.record(z.string()).nullable(),
142
-
143
- // log line emitted by the unit
144
- logLine: z.string().nullable(),
145
- })
146
- .partial()
147
-
148
- export type InstanceStatusFieldValue = z.infer<typeof instanceStatusFieldValueSchema>
149
- export type InstanceStatusField = z.infer<typeof instanceStatusFieldSchema>
150
- export type InstanceTerminal = z.infer<typeof instanceTerminalSchema>
151
-
152
- export type InstanceStatus = z.infer<typeof instanceStatusSchema>
153
- export type InstanceState = z.infer<typeof instanceStateSchema>
154
-
155
- export type InstanceStateUpdate = z.infer<typeof instanceStateUpdateSchema>
156
-
157
- export type InstancePageBlock = z.infer<typeof instancePageBlockSchema>
158
- export type InstancePageMeta = z.infer<typeof instancePageMetaSchema>
159
- export type InstanceFileMeta = z.infer<typeof instanceFileMetaSchema>
160
-
161
- export type InstanceTriggerSpec = z.infer<typeof instanceTriggerSpecSchema>
162
- export type InstanceTrigger = z.infer<typeof instanceTriggerSchema>
163
- export type InstanceTriggerInvocation = z.infer<typeof instanceTriggerInvocationSchema>
164
-
165
- export function createInstanceState(
166
- id: string,
167
- status: InstanceStatus = "not_created",
168
- fields: Partial<InstanceState> = {},
169
- ): InstanceState {
170
- return {
171
- id,
172
- parentId: null,
173
- dependencyIds: [],
174
- status: status,
175
-
176
- latestOperationId: null,
177
-
178
- currentResourceCount: null,
179
- totalResourceCount: null,
180
- inputHash: null,
181
- outputHash: null,
182
-
183
- error: null,
184
- evaluationError: null,
185
-
186
- statusFields: [],
187
- files: [],
188
- pages: [],
189
- terminals: [],
190
- triggers: [],
191
-
192
- ...fields,
193
- }
194
- }
195
-
196
- export function applyPartialInstanceState(
197
- states: Map<string, InstanceState>,
198
- patch: Partial<InstanceState>,
199
- ): InstanceState {
200
- if (!patch.id) {
201
- throw new Error("The ID of the instance state is required.")
202
- }
203
-
204
- let state = states.get(patch.id) ?? createInstanceState(patch.id, "unknown")
205
- state = { ...state, ...patch }
206
-
207
- states.set(state.id, state)
208
- return state
209
- }
210
-
211
- export function createInstanceStatePatch(update: InstanceStateUpdate): Partial<InstanceState> {
212
- return omit(update, ["secrets", "logLine"])
213
- }
214
-
215
- /**
216
- * Builds a map where instance id -> instance state ids that directly depend on the instance.
217
- *
218
- * @param instanceStates The instance states to build the map from.
219
- */
220
- export function buildDependentInstanceStateMap(
221
- instanceStates: Map<string, InstanceState>,
222
- ): Map<string, string[]> {
223
- const dependentMap = new Map<string, string[]>()
224
-
225
- for (const state of instanceStates.values()) {
226
- for (const dependency of state.dependencyIds) {
227
- let dependents = dependentMap.get(dependency)
228
- if (!dependents) {
229
- dependents = []
230
- dependentMap.set(dependency, dependents)
231
- }
232
-
233
- dependents.push(state.id)
234
- }
235
- }
236
-
237
- return dependentMap
238
- }
239
-
240
- /**
241
- * Gets all instance ids that depend on the given instance id (including recursively).
242
- * The instance id itself is not included in the result.
243
- *
244
- * @param dependentMap The dependent map.
245
- * @param instanceId The instance id to get all dependent instance ids.
246
- */
247
- export function getAllDependentInstanceIds(
248
- dependentMap: Map<string, string[]>,
249
- instanceId: string,
250
- ): string[] {
251
- const result = new Set<string>()
252
- const visited = new Set<string>()
253
-
254
- const traverse = (id: string) => {
255
- if (visited.has(id)) return
256
- visited.add(id)
257
-
258
- const dependents = dependentMap.get(id)
259
- if (!dependents) return
260
-
261
- for (const dependent of dependents) {
262
- result.add(dependent)
263
- traverse(dependent)
264
- }
265
- }
266
-
267
- traverse(instanceId)
268
-
269
- return Array.from(result)
270
- }
@@ -1,13 +0,0 @@
1
- import { z } from "zod"
2
-
3
- export const terminalSessionSchema = z.object({
4
- id: z.string().nanoid(),
5
- projectId: z.string(),
6
- instanceId: z.string(),
7
- terminalName: z.string(),
8
- terminalTitle: z.string(),
9
- createdAt: z.coerce.date(),
10
- finishedAt: z.coerce.date().optional(),
11
- })
12
-
13
- export type TerminalSession = z.infer<typeof terminalSessionSchema>