@nylorun/runtime 0.15.1-beta → 0.16.0-beta

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (220) hide show
  1. package/CHANGELOG.md +82 -0
  2. package/README.md +33 -6
  3. package/dist/adapters/execution/restate.d.ts +9 -1
  4. package/dist/adapters/execution/restate.js +64 -5
  5. package/dist/adapters/replication/pgoutput.js +3 -1
  6. package/dist/adapters/sandbox/local.d.ts +10 -0
  7. package/dist/adapters/sandbox/local.js +195 -0
  8. package/dist/adapters/sandbox/virtual.js +36 -1
  9. package/dist/admin-openapi.json +37 -3
  10. package/dist/api/a2a/tasks.js +6 -1
  11. package/dist/api/ag-ui/history.js +6 -1
  12. package/dist/api/ag-ui/routes.js +5 -1
  13. package/dist/api/components.d.ts +2673 -655
  14. package/dist/api/components.js +17 -1
  15. package/dist/api/gate/keys.d.ts +4 -3
  16. package/dist/api/gate/routes.d.ts +19 -3
  17. package/dist/api/gate/routes.js +162 -53
  18. package/dist/api/http/app.js +4 -0
  19. package/dist/api/http/define.d.ts +10 -0
  20. package/dist/api/http/define.js +2 -1
  21. package/dist/api/http/routes/artifacts.d.ts +11 -0
  22. package/dist/api/http/routes/artifacts.js +518 -0
  23. package/dist/api/http/routes/sandboxes.d.ts +9 -0
  24. package/dist/api/http/routes/sandboxes.js +152 -0
  25. package/dist/api/http/routes/sessions.js +3 -1
  26. package/dist/api/http/routes/tenant.js +36 -2
  27. package/dist/artifacts/config.d.ts +23 -0
  28. package/dist/artifacts/config.js +27 -0
  29. package/dist/artifacts/export.d.ts +8 -0
  30. package/dist/artifacts/export.js +205 -0
  31. package/dist/artifacts/files.d.ts +36 -0
  32. package/dist/artifacts/files.js +45 -0
  33. package/dist/artifacts/folders.d.ts +60 -0
  34. package/dist/artifacts/folders.js +209 -0
  35. package/dist/artifacts/links.d.ts +28 -0
  36. package/dist/artifacts/links.js +92 -0
  37. package/dist/artifacts/media-types.d.ts +13 -0
  38. package/dist/artifacts/media-types.js +80 -0
  39. package/dist/artifacts/parts.d.ts +34 -0
  40. package/dist/artifacts/parts.js +38 -0
  41. package/dist/artifacts/service.d.ts +74 -0
  42. package/dist/artifacts/service.js +301 -0
  43. package/dist/artifacts/workspace.d.ts +60 -0
  44. package/dist/artifacts/workspace.js +57 -0
  45. package/dist/artifacts/zip.d.ts +10 -0
  46. package/dist/artifacts/zip.js +126 -0
  47. package/dist/blob/fs.d.ts +13 -0
  48. package/dist/blob/fs.js +194 -0
  49. package/dist/blob/index.d.ts +9 -0
  50. package/dist/blob/index.js +9 -0
  51. package/dist/blob/s3.d.ts +42 -0
  52. package/dist/blob/s3.js +366 -0
  53. package/dist/blob/types.d.ts +108 -0
  54. package/dist/blob/types.js +85 -0
  55. package/dist/core/sandbox-routes.d.ts +2 -2
  56. package/dist/core/sandbox-routes.js +4 -3
  57. package/dist/execution/memory.d.ts +11 -1
  58. package/dist/execution/memory.js +77 -3
  59. package/dist/execution/types.d.ts +54 -1
  60. package/dist/execution/types.js +31 -0
  61. package/dist/gates/contract.d.ts +14 -13
  62. package/dist/gates/contract.js +13 -12
  63. package/dist/gates/egress.d.ts +72 -0
  64. package/dist/gates/egress.js +319 -0
  65. package/dist/gates/handler.d.ts +3 -0
  66. package/dist/gates/handler.js +3 -2
  67. package/dist/gates/http-client.d.ts +3 -2
  68. package/dist/gates/http-client.js +17 -6
  69. package/dist/gates/in-process.d.ts +13 -1
  70. package/dist/gates/in-process.js +11 -0
  71. package/dist/gates/inflight.d.ts +23 -4
  72. package/dist/gates/inflight.js +18 -4
  73. package/dist/gates/model-gate.d.ts +6 -2
  74. package/dist/gates/tool-calls.d.ts +9 -5
  75. package/dist/gates/tool-calls.js +6 -6
  76. package/dist/gates/tool-client.d.ts +9 -2
  77. package/dist/gates/tool-client.js +56 -22
  78. package/dist/gates/tool-contract.d.ts +12 -7
  79. package/dist/gates/tool-contract.js +10 -4
  80. package/dist/gates/tool-gate.d.ts +1 -0
  81. package/dist/harness/calls.d.ts +52 -0
  82. package/dist/harness/calls.js +113 -0
  83. package/dist/harness/executors.d.ts +22 -0
  84. package/dist/harness/executors.js +96 -0
  85. package/dist/harness/main.d.ts +6 -0
  86. package/dist/harness/main.js +103 -0
  87. package/dist/harness/pod.d.ts +21 -0
  88. package/dist/harness/pod.js +119 -0
  89. package/dist/harness/run-tokens.d.ts +18 -0
  90. package/dist/harness/run-tokens.js +15 -0
  91. package/dist/harness/service.d.ts +46 -0
  92. package/dist/harness/service.js +179 -0
  93. package/dist/harness/ws-client.d.ts +43 -0
  94. package/dist/harness/ws-client.js +122 -0
  95. package/dist/harness/ws-port.d.ts +22 -0
  96. package/dist/harness/ws-port.js +78 -0
  97. package/dist/harness-api/in-process.d.ts +24 -0
  98. package/dist/harness-api/in-process.js +119 -0
  99. package/dist/harness-api/record.d.ts +44 -0
  100. package/dist/harness-api/record.js +250 -0
  101. package/dist/harness-api/renew.d.ts +37 -0
  102. package/dist/harness-api/renew.js +77 -0
  103. package/dist/harness-api/server.d.ts +104 -0
  104. package/dist/harness-api/server.js +408 -0
  105. package/dist/harness-api/start.d.ts +36 -0
  106. package/dist/harness-api/start.js +118 -0
  107. package/dist/harness-api/workspace.d.ts +53 -0
  108. package/dist/harness-api/workspace.js +201 -0
  109. package/dist/harness-api/ws-server.d.ts +34 -0
  110. package/dist/harness-api/ws-server.js +227 -0
  111. package/dist/host/app.js +23 -2
  112. package/dist/host/gates.d.ts +27 -2
  113. package/dist/host/gates.js +46 -1
  114. package/dist/host/http.d.ts +4 -1
  115. package/dist/host/http.js +6 -1
  116. package/dist/host/main.js +135 -27
  117. package/dist/host/stack-config.d.ts +101 -1
  118. package/dist/host/stack-config.js +199 -8
  119. package/dist/mcp/snapshot.d.ts +8 -1
  120. package/dist/mcp/snapshot.js +28 -0
  121. package/dist/media.d.ts +5 -24
  122. package/dist/media.js +5 -0
  123. package/dist/model/pi-model.d.ts +6 -2
  124. package/dist/model/pi-model.js +39 -22
  125. package/dist/node/index.d.ts +1 -1
  126. package/dist/node/index.js +1 -1
  127. package/dist/openapi.json +15245 -8850
  128. package/dist/record/index.d.ts +1 -0
  129. package/dist/record/index.js +1 -0
  130. package/dist/record/sandbox.d.ts +25 -0
  131. package/dist/record/sandbox.js +43 -0
  132. package/dist/sandbox/capability.d.ts +3 -0
  133. package/dist/sandbox/capability.js +10 -0
  134. package/dist/sandbox/egress-token.d.ts +26 -0
  135. package/dist/sandbox/egress-token.js +8 -0
  136. package/dist/sandbox/join.d.ts +37 -0
  137. package/dist/sandbox/join.js +165 -0
  138. package/dist/sandbox/manager.d.ts +99 -34
  139. package/dist/sandbox/manager.js +216 -84
  140. package/dist/sandbox/placement.d.ts +27 -0
  141. package/dist/sandbox/placement.js +44 -0
  142. package/dist/sandbox/pods/client.d.ts +90 -0
  143. package/dist/sandbox/pods/client.js +79 -0
  144. package/dist/sandbox/pods/lifecycle.d.ts +91 -0
  145. package/dist/sandbox/pods/lifecycle.js +178 -0
  146. package/dist/sandbox/pods/name.d.ts +1 -0
  147. package/dist/sandbox/pods/name.js +28 -0
  148. package/dist/sandbox/pods/network-gate.d.ts +25 -0
  149. package/dist/sandbox/pods/network-gate.js +71 -0
  150. package/dist/sandbox/pods/reconcile.d.ts +11 -0
  151. package/dist/sandbox/pods/reconcile.js +152 -0
  152. package/dist/sandbox/pods/spec.d.ts +45 -0
  153. package/dist/sandbox/pods/spec.js +62 -0
  154. package/dist/sandbox/records.d.ts +29 -0
  155. package/dist/sandbox/records.js +79 -0
  156. package/dist/sandbox/resolve.d.ts +5 -0
  157. package/dist/sandbox/resolve.js +3 -2
  158. package/dist/sandbox/session-sandbox.js +7 -5
  159. package/dist/sandbox/share.d.ts +11 -0
  160. package/dist/sandbox/share.js +10 -0
  161. package/dist/sandbox/tenant-config.d.ts +16 -1
  162. package/dist/sandbox/tenant-config.js +9 -0
  163. package/dist/sandbox/types.d.ts +29 -1
  164. package/dist/sandbox/types.js +13 -1
  165. package/dist/store/postgres/drizzle/0004_sandbox_resources.sql +20 -0
  166. package/dist/store/postgres/drizzle/0005_artifacts.sql +27 -0
  167. package/dist/store/postgres/drizzle/0006_folder_artifacts.sql +12 -0
  168. package/dist/store/postgres/drizzle/0007_sandbox_pods.sql +14 -0
  169. package/dist/store/postgres/drizzle/meta/_journal.json +28 -0
  170. package/dist/store/postgres/record-writer.d.ts +6 -1
  171. package/dist/store/postgres/record-writer.js +20 -2
  172. package/dist/store/postgres/schema.d.ts +894 -0
  173. package/dist/store/postgres/schema.js +130 -2
  174. package/dist/store/postgres/store.js +289 -4
  175. package/dist/store/types.d.ts +143 -6
  176. package/dist/streams/control.d.ts +2 -0
  177. package/dist/streams/control.js +5 -0
  178. package/dist/streams/types.d.ts +10 -1
  179. package/dist/tenant/advance.js +218 -160
  180. package/dist/tenant/artifact-tool.d.ts +5 -0
  181. package/dist/tenant/artifact-tool.js +119 -0
  182. package/dist/tenant/commands.js +30 -5
  183. package/dist/tenant/context.d.ts +64 -5
  184. package/dist/tenant/effects.d.ts +19 -13
  185. package/dist/tenant/effects.js +5 -382
  186. package/dist/tenant/history.d.ts +13 -23
  187. package/dist/tenant/history.js +14 -62
  188. package/dist/tenant/host-token.d.ts +92 -0
  189. package/dist/tenant/host-token.js +179 -0
  190. package/dist/tenant/module.js +1 -0
  191. package/dist/tenant/paths.d.ts +2 -1
  192. package/dist/tenant/paths.js +3 -1
  193. package/dist/tenant/reset.d.ts +5 -2
  194. package/dist/tenant/reset.js +22 -2
  195. package/dist/tenant/run-grants.d.ts +49 -0
  196. package/dist/tenant/run-grants.js +62 -0
  197. package/dist/tenant/run-token.d.ts +70 -0
  198. package/dist/tenant/run-token.js +138 -0
  199. package/dist/tenant/runtime.d.ts +38 -0
  200. package/dist/tenant/runtime.js +120 -32
  201. package/dist/tenant/sandboxes.d.ts +119 -0
  202. package/dist/tenant/sandboxes.js +551 -0
  203. package/dist/tenant/session.d.ts +2 -13
  204. package/dist/tenant/session.js +4 -28
  205. package/dist/tenant/sessions.d.ts +5 -0
  206. package/dist/tenant/sessions.js +28 -1
  207. package/dist/tenant/status.d.ts +8 -3
  208. package/dist/tenant/status.js +5 -0
  209. package/dist/tenant/streams.d.ts +7 -0
  210. package/dist/tenant/streams.js +24 -1
  211. package/dist/tenant/sweep.js +37 -4
  212. package/dist/tenant/tokens.js +9 -2
  213. package/dist/tenant/types.d.ts +18 -1
  214. package/dist/tenant/worker.d.ts +11 -5
  215. package/dist/tenant/worker.js +12 -6
  216. package/dist/version.d.ts +1 -1
  217. package/dist/version.js +1 -1
  218. package/package.json +13 -10
  219. package/dist/adapters/media.d.ts +0 -24
  220. package/dist/adapters/media.js +0 -111
@@ -0,0 +1,194 @@
1
+ import { createHash, randomUUID } from "node:crypto";
2
+ import { mkdir, open, readFile, readdir, rename, rm, stat, unlink } from "node:fs/promises";
3
+ import { dirname, join, relative, sep } from "node:path";
4
+ import { Readable } from "node:stream";
5
+ import { BlobRangeError, BlobTooLargeError, DEFAULT_CONTENT_TYPE, assertBlobKey, assertBlobPrefix, assertMaxBytes, assertRange, chunksOf, } from "./types.js";
6
+ function isMissing(error) {
7
+ const code = error?.code;
8
+ return code === "ENOENT" || code === "ENOTDIR";
9
+ }
10
+ /**
11
+ * The `fs` adapter: objects as files under `root`, for embedding (`startEphemeralRuntime`, a
12
+ * Host without an Object store) and tests. `objects/<key>` holds the bytes, `meta/<key>.json`
13
+ * the content type; a write goes to `tmp/` first and is renamed into place, so a reader sees
14
+ * the old object or the new one, never part of one. A key cannot also be the directory of
15
+ * another key (`a` and `a/b`); the Runtime's keys never are.
16
+ */
17
+ export function createFsBlobStore(options) {
18
+ const objects = join(options.root, "objects");
19
+ const metadata = join(options.root, "meta");
20
+ const tmp = join(options.root, "tmp");
21
+ const objectPath = (key) => join(objects, ...key.split("/"));
22
+ const metaPath = (key) => `${join(metadata, ...key.split("/"))}.json`;
23
+ async function readMeta(key) {
24
+ try {
25
+ const parsed = JSON.parse(await readFile(metaPath(key), "utf8"));
26
+ return { contentType: parsed.contentType ?? DEFAULT_CONTENT_TYPE };
27
+ }
28
+ catch {
29
+ return { contentType: DEFAULT_CONTENT_TYPE };
30
+ }
31
+ }
32
+ async function head(key) {
33
+ assertBlobKey(key);
34
+ let stats;
35
+ try {
36
+ stats = await stat(objectPath(key));
37
+ }
38
+ catch (error) {
39
+ if (isMissing(error))
40
+ return undefined;
41
+ throw error;
42
+ }
43
+ if (!stats.isFile())
44
+ return undefined;
45
+ const meta = await readMeta(key);
46
+ return { key, size: stats.size, contentType: meta.contentType, lastModified: stats.mtime };
47
+ }
48
+ return {
49
+ kind: "fs",
50
+ async put(key, body, put = {}) {
51
+ assertBlobKey(key);
52
+ assertMaxBytes(put.maxBytes);
53
+ put.signal?.throwIfAborted();
54
+ await mkdir(tmp, { recursive: true });
55
+ const id = randomUUID();
56
+ const dataTmp = join(tmp, id);
57
+ const metaTmp = join(tmp, `${id}.json`);
58
+ const hash = createHash("sha256");
59
+ let size = 0;
60
+ const file = await open(dataTmp, "wx", 0o600);
61
+ try {
62
+ try {
63
+ for await (const chunk of chunksOf(body)) {
64
+ put.signal?.throwIfAborted();
65
+ size += chunk.byteLength;
66
+ if (put.maxBytes !== undefined && size > put.maxBytes)
67
+ throw new BlobTooLargeError(put.maxBytes);
68
+ hash.update(chunk);
69
+ await file.write(chunk);
70
+ }
71
+ await file.sync();
72
+ }
73
+ finally {
74
+ await file.close();
75
+ }
76
+ const stored = { contentType: put.contentType ?? DEFAULT_CONTENT_TYPE };
77
+ await writeTmp(metaTmp, JSON.stringify(stored));
78
+ await mkdir(dirname(objectPath(key)), { recursive: true });
79
+ await mkdir(dirname(metaPath(key)), { recursive: true });
80
+ await rename(metaTmp, metaPath(key));
81
+ await rename(dataTmp, objectPath(key));
82
+ }
83
+ catch (error) {
84
+ await rm(dataTmp, { force: true });
85
+ await rm(metaTmp, { force: true });
86
+ throw error;
87
+ }
88
+ return { key, size, sha256: hash.digest("hex") };
89
+ },
90
+ async get(key, get = {}) {
91
+ assertBlobKey(key);
92
+ if (get.range)
93
+ assertRange(get.range);
94
+ get.signal?.throwIfAborted();
95
+ let file;
96
+ try {
97
+ file = await open(objectPath(key), "r");
98
+ }
99
+ catch (error) {
100
+ if (isMissing(error) || error?.code === "EISDIR")
101
+ return undefined;
102
+ throw error;
103
+ }
104
+ try {
105
+ const stats = await file.stat();
106
+ if (!stats.isFile()) {
107
+ await file.close();
108
+ return undefined;
109
+ }
110
+ const meta = await readMeta(key);
111
+ const base = {
112
+ key,
113
+ size: stats.size,
114
+ contentType: meta.contentType,
115
+ lastModified: stats.mtime,
116
+ };
117
+ if (!get.range) {
118
+ const stream = file.createReadStream({ autoClose: true });
119
+ return { ...base, body: Readable.toWeb(stream) };
120
+ }
121
+ if (get.range.start >= stats.size)
122
+ throw new BlobRangeError(get.range, stats.size);
123
+ const end = Math.min(get.range.end ?? stats.size - 1, stats.size - 1);
124
+ const stream = file.createReadStream({ start: get.range.start, end, autoClose: true });
125
+ return {
126
+ ...base,
127
+ body: Readable.toWeb(stream),
128
+ range: { start: get.range.start, end },
129
+ };
130
+ }
131
+ catch (error) {
132
+ await file.close().catch(() => undefined);
133
+ throw error;
134
+ }
135
+ },
136
+ head,
137
+ async delete(key) {
138
+ assertBlobKey(key);
139
+ for (const path of [objectPath(key), metaPath(key)])
140
+ try {
141
+ await unlink(path);
142
+ }
143
+ catch (error) {
144
+ const code = error?.code;
145
+ if (!isMissing(error) && code !== "EISDIR" && code !== "EPERM")
146
+ throw error;
147
+ }
148
+ },
149
+ async *list(prefix) {
150
+ assertBlobPrefix(prefix);
151
+ // Walk the deepest directory the prefix names, then filter and sort: `a-b` sorts before
152
+ // `a/c`, so directory order is not key order.
153
+ const slash = prefix.lastIndexOf("/");
154
+ const start = slash < 0 ? objects : join(objects, ...prefix.slice(0, slash).split("/"));
155
+ const found = [];
156
+ const walk = async (directory) => {
157
+ let entries;
158
+ try {
159
+ entries = await readdir(directory, { withFileTypes: true });
160
+ }
161
+ catch (error) {
162
+ if (isMissing(error))
163
+ return;
164
+ throw error;
165
+ }
166
+ for (const entry of entries) {
167
+ const path = join(directory, entry.name);
168
+ if (entry.isDirectory())
169
+ await walk(path);
170
+ else if (entry.isFile()) {
171
+ const key = relative(objects, path).split(sep).join("/");
172
+ if (!key.startsWith(prefix))
173
+ continue;
174
+ const stats = await stat(path).catch(() => undefined);
175
+ if (stats)
176
+ found.push({ key, size: stats.size, lastModified: stats.mtime });
177
+ }
178
+ }
179
+ };
180
+ await walk(start);
181
+ found.sort((left, right) => (left.key < right.key ? -1 : left.key > right.key ? 1 : 0));
182
+ yield* found;
183
+ },
184
+ };
185
+ }
186
+ async function writeTmp(path, text) {
187
+ const file = await open(path, "wx", 0o600);
188
+ try {
189
+ await file.writeFile(text, "utf8");
190
+ }
191
+ finally {
192
+ await file.close();
193
+ }
194
+ }
@@ -0,0 +1,9 @@
1
+ /**
2
+ * The `BlobStore` seam (`./types.ts`) and its adapters. A Host with an Object store
3
+ * (`StackConfig.objectStore`, from `NYLORUN_OBJECT_STORE_*`) passes an `s3` store to its Tenant
4
+ * (`TenantOpenHooks.blobs`); without one the Tenant keeps blobs on disk (`fs`, under
5
+ * `TenantPaths.blobs`). Tenant code reaches it as `TenantContext.blobs`.
6
+ */
7
+ export * from "./types.js";
8
+ export { createFsBlobStore, type FsBlobStoreOptions } from "./fs.js";
9
+ export { createS3BlobStore, type S3BlobStore, type S3BlobStoreOptions } from "./s3.js";
@@ -0,0 +1,9 @@
1
+ /**
2
+ * The `BlobStore` seam (`./types.ts`) and its adapters. A Host with an Object store
3
+ * (`StackConfig.objectStore`, from `NYLORUN_OBJECT_STORE_*`) passes an `s3` store to its Tenant
4
+ * (`TenantOpenHooks.blobs`); without one the Tenant keeps blobs on disk (`fs`, under
5
+ * `TenantPaths.blobs`). Tenant code reaches it as `TenantContext.blobs`.
6
+ */
7
+ export * from "./types.js";
8
+ export { createFsBlobStore } from "./fs.js";
9
+ export { createS3BlobStore } from "./s3.js";
@@ -0,0 +1,42 @@
1
+ import { type BlobStore } from "./types.js";
2
+ export interface S3BlobStoreOptions {
3
+ /** The S3 endpoint, e.g. `http://rustfs:9000` or `https://s3.eu-west-1.amazonaws.com`. */
4
+ endpoint: string;
5
+ bucket: string;
6
+ /** The signing region. Default `us-east-1`, which RustFS accepts. */
7
+ region?: string;
8
+ accessKeyId: string;
9
+ secretAccessKey: string;
10
+ /**
11
+ * Path-style URLs (`<endpoint>/<bucket>/<key>`), which RustFS and most S3 servers expect.
12
+ * `false` uses virtual-hosted URLs (`<bucket>.<endpoint host>/<key>`). Default `true`.
13
+ */
14
+ forcePathStyle?: boolean;
15
+ /**
16
+ * A body longer than this is uploaded in parts of about this size (multipart upload); a
17
+ * shorter one in one request. At least 5 MiB, the S3 minimum. Default 8 MiB.
18
+ */
19
+ partSize?: number;
20
+ /** Attempts for a request that fails with a network error, 429 or 5xx. Default 3. */
21
+ attempts?: number;
22
+ /** Replaces the global `fetch` (tests). */
23
+ fetch?: typeof fetch;
24
+ }
25
+ /** The `s3` adapter, plus the one bucket operation the Runtime needs at boot. */
26
+ export interface S3BlobStore extends BlobStore {
27
+ readonly kind: "s3";
28
+ /** Create the bucket unless it exists. The local stack's runtime calls it at boot. */
29
+ ensureBucket(): Promise<void>;
30
+ }
31
+ /**
32
+ * The `s3` adapter over the plain S3 API: PutObject, multipart upload, GetObject with Range,
33
+ * HeadObject, DeleteObject and ListObjectsV2, signed with SigV4 by aws4fetch (MIT, no
34
+ * dependencies, built on `fetch` and web streams). It uses no server's admin, KMS or lifecycle
35
+ * API (D35), so RustFS, S3, R2 or any S3 server sits behind it.
36
+ *
37
+ * Uploads stream: the body is counted and hashed as it arrives and sent in parts of
38
+ * `partSize`, so memory stays at about one part whatever the size, and a body that passes
39
+ * `maxBytes` aborts its multipart upload. Each request carries the SHA-256 of its payload,
40
+ * which the server checks. Downloads hand back the response's stream.
41
+ */
42
+ export declare function createS3BlobStore(options: S3BlobStoreOptions): S3BlobStore;
@@ -0,0 +1,366 @@
1
+ import { createHash } from "node:crypto";
2
+ import { AwsClient } from "aws4fetch";
3
+ import { BlobRangeError, BlobStoreError, BlobTooLargeError, DEFAULT_CONTENT_TYPE, assertBlobKey, assertBlobPrefix, assertMaxBytes, assertRange, chunksOf, } from "./types.js";
4
+ const MIN_PART_SIZE = 5 * 1024 * 1024;
5
+ const DEFAULT_PART_SIZE = 8 * 1024 * 1024;
6
+ function sha256Hex(bytes) {
7
+ return createHash("sha256").update(bytes).digest("hex");
8
+ }
9
+ function concat(chunks, length) {
10
+ if (chunks.length === 1)
11
+ return chunks[0];
12
+ const out = new Uint8Array(length);
13
+ let offset = 0;
14
+ for (const chunk of chunks) {
15
+ out.set(chunk, offset);
16
+ offset += chunk.byteLength;
17
+ }
18
+ return out;
19
+ }
20
+ function decodeXml(text) {
21
+ return text
22
+ .replace(/&lt;/g, "<")
23
+ .replace(/&gt;/g, ">")
24
+ .replace(/&quot;/g, '"')
25
+ .replace(/&apos;/g, "'")
26
+ .replace(/&#(\d+);/g, (_, code) => String.fromCodePoint(Number(code)))
27
+ .replace(/&amp;/g, "&");
28
+ }
29
+ function escapeXml(text) {
30
+ return text.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;");
31
+ }
32
+ /** The text of the first `<name>` element in `xml`, decoded. */
33
+ function tag(xml, name) {
34
+ const match = new RegExp(`<${name}>([\\s\\S]*?)</${name}>`).exec(xml);
35
+ return match ? decodeXml(match[1]) : undefined;
36
+ }
37
+ function blocks(xml, name) {
38
+ return [...xml.matchAll(new RegExp(`<${name}>([\\s\\S]*?)</${name}>`, "g"))].map((m) => m[1]);
39
+ }
40
+ /**
41
+ * The `s3` adapter over the plain S3 API: PutObject, multipart upload, GetObject with Range,
42
+ * HeadObject, DeleteObject and ListObjectsV2, signed with SigV4 by aws4fetch (MIT, no
43
+ * dependencies, built on `fetch` and web streams). It uses no server's admin, KMS or lifecycle
44
+ * API (D35), so RustFS, S3, R2 or any S3 server sits behind it.
45
+ *
46
+ * Uploads stream: the body is counted and hashed as it arrives and sent in parts of
47
+ * `partSize`, so memory stays at about one part whatever the size, and a body that passes
48
+ * `maxBytes` aborts its multipart upload. Each request carries the SHA-256 of its payload,
49
+ * which the server checks. Downloads hand back the response's stream.
50
+ */
51
+ export function createS3BlobStore(options) {
52
+ const endpoint = new URL(options.endpoint);
53
+ if (endpoint.protocol !== "http:" && endpoint.protocol !== "https:")
54
+ throw new TypeError("The S3 endpoint must be an http or https URL");
55
+ const base = endpoint.toString().replace(/\/+$/, "");
56
+ const pathStyle = options.forcePathStyle !== false;
57
+ const bucketUrl = pathStyle
58
+ ? `${base}/${options.bucket}`
59
+ : `${endpoint.protocol}//${options.bucket}.${endpoint.host}${endpoint.pathname.replace(/\/+$/, "")}`;
60
+ const partSize = Math.max(options.partSize ?? DEFAULT_PART_SIZE, MIN_PART_SIZE);
61
+ const attempts = Math.max(1, options.attempts ?? 3);
62
+ const fetchImpl = options.fetch ?? globalThis.fetch;
63
+ const client = new AwsClient({
64
+ accessKeyId: options.accessKeyId,
65
+ secretAccessKey: options.secretAccessKey,
66
+ service: "s3",
67
+ region: options.region ?? "us-east-1",
68
+ retries: 0,
69
+ });
70
+ const objectUrl = (key, query) => {
71
+ const url = new URL(`${bucketUrl}/${key}`);
72
+ for (const [name, value] of Object.entries(query ?? {}))
73
+ url.searchParams.set(name, value);
74
+ return url.toString();
75
+ };
76
+ /** One signed request; retried on a network error, 429 or 5xx unless `once`. */
77
+ async function send(url, init, once = false) {
78
+ let lastError;
79
+ for (let attempt = 1; attempt <= (once ? 1 : attempts); attempt += 1) {
80
+ try {
81
+ const response = await fetchImpl(await client.sign(url, init));
82
+ if ((response.status >= 500 || response.status === 429) && attempt < attempts && !once) {
83
+ await response.body?.cancel();
84
+ await new Promise((resolve) => setTimeout(resolve, 50 * 2 ** attempt));
85
+ continue;
86
+ }
87
+ return response;
88
+ }
89
+ catch (error) {
90
+ if (init.signal?.aborted)
91
+ throw error;
92
+ lastError = error;
93
+ if (attempt < (once ? 1 : attempts))
94
+ await new Promise((resolve) => setTimeout(resolve, 50 * 2 ** attempt));
95
+ }
96
+ }
97
+ throw new BlobStoreError(`The Object store is unreachable: ${lastError instanceof Error ? lastError.message : String(lastError)}`);
98
+ }
99
+ async function failure(response, what) {
100
+ const text = await response.text().catch(() => "");
101
+ const code = tag(text, "Code");
102
+ const message = tag(text, "Message");
103
+ return new BlobStoreError(`${what} failed: ${response.status}${code ? ` ${code}` : ""}${message ? `: ${message}` : ""}`, response.status, code);
104
+ }
105
+ /** PUT with the payload's SHA-256, which the server verifies. */
106
+ const putBytes = (url, bytes, headers, signal) => send(url, {
107
+ method: "PUT",
108
+ // fetch sends any ArrayBufferView; the DOM types want one over an ArrayBuffer.
109
+ body: bytes,
110
+ headers: { ...headers, "x-amz-content-sha256": sha256Hex(bytes) },
111
+ ...(signal ? { signal } : {}),
112
+ });
113
+ async function head(key) {
114
+ assertBlobKey(key);
115
+ const response = await send(objectUrl(key), { method: "HEAD" });
116
+ if (response.status === 404)
117
+ return undefined;
118
+ if (!response.ok)
119
+ throw await failure(response, `HEAD ${key}`);
120
+ return metaOf(key, response.headers, Number(response.headers.get("content-length") ?? 0));
121
+ }
122
+ function metaOf(key, headers, size) {
123
+ const modified = headers.get("last-modified");
124
+ return {
125
+ key,
126
+ size,
127
+ contentType: headers.get("content-type") ?? DEFAULT_CONTENT_TYPE,
128
+ lastModified: modified ? new Date(modified) : new Date(0),
129
+ };
130
+ }
131
+ async function putMultipart(key, first, rest, state, put) {
132
+ const signal = put.signal;
133
+ const created = await send(objectUrl(key, { uploads: "" }), {
134
+ method: "POST",
135
+ headers: { "content-type": put.contentType ?? DEFAULT_CONTENT_TYPE },
136
+ ...(signal ? { signal } : {}),
137
+ }, true);
138
+ if (!created.ok)
139
+ throw await failure(created, `CreateMultipartUpload ${key}`);
140
+ const uploadId = tag(await created.text(), "UploadId");
141
+ if (!uploadId)
142
+ throw new BlobStoreError(`CreateMultipartUpload ${key} returned no UploadId`);
143
+ const parts = [];
144
+ const uploadPart = async (bytes) => {
145
+ const number = parts.length + 1;
146
+ const response = await putBytes(objectUrl(key, { partNumber: String(number), uploadId }), bytes, {}, signal);
147
+ if (!response.ok)
148
+ throw await failure(response, `UploadPart ${number} of ${key}`);
149
+ await response.body?.cancel();
150
+ const etag = response.headers.get("etag");
151
+ if (!etag)
152
+ throw new BlobStoreError(`UploadPart ${number} of ${key} returned no ETag`);
153
+ parts.push({ number, etag });
154
+ };
155
+ try {
156
+ await uploadPart(first);
157
+ let buffered = [];
158
+ let length = 0;
159
+ for (;;) {
160
+ const next = await rest.next();
161
+ if (next.done)
162
+ break;
163
+ const chunk = next.value;
164
+ signal?.throwIfAborted();
165
+ state.size += chunk.byteLength;
166
+ if (put.maxBytes !== undefined && state.size > put.maxBytes)
167
+ throw new BlobTooLargeError(put.maxBytes);
168
+ state.hash.update(chunk);
169
+ buffered.push(chunk);
170
+ length += chunk.byteLength;
171
+ if (length >= partSize) {
172
+ await uploadPart(concat(buffered, length));
173
+ buffered = [];
174
+ length = 0;
175
+ }
176
+ }
177
+ if (length > 0)
178
+ await uploadPart(concat(buffered, length));
179
+ const body = `<CompleteMultipartUpload>${parts
180
+ .map((part) => `<Part><PartNumber>${part.number}</PartNumber><ETag>${escapeXml(part.etag)}</ETag></Part>`)
181
+ .join("")}</CompleteMultipartUpload>`;
182
+ const completed = await send(objectUrl(key, { uploadId }), {
183
+ method: "POST",
184
+ body,
185
+ headers: { "content-type": "application/xml", "x-amz-content-sha256": sha256Hex(new TextEncoder().encode(body)) },
186
+ ...(signal ? { signal } : {}),
187
+ }, true);
188
+ // CompleteMultipartUpload may answer 200 with an error in the body.
189
+ const text = completed.ok ? await completed.text() : "";
190
+ if (!completed.ok)
191
+ throw await failure(completed, `CompleteMultipartUpload ${key}`);
192
+ if (/<Error>/.test(text))
193
+ throw new BlobStoreError(`CompleteMultipartUpload ${key} failed: ${tag(text, "Code") ?? "error"}: ${tag(text, "Message") ?? ""}`, completed.status, tag(text, "Code"));
194
+ }
195
+ catch (error) {
196
+ // Best effort: an upload left behind holds only parts, never an object.
197
+ await send(objectUrl(key, { uploadId }), { method: "DELETE" })
198
+ .then((response) => response.body?.cancel())
199
+ .catch(() => undefined);
200
+ throw error;
201
+ }
202
+ }
203
+ return {
204
+ kind: "s3",
205
+ async ensureBucket() {
206
+ const found = await send(bucketUrl, { method: "HEAD" });
207
+ if (found.ok)
208
+ return;
209
+ if (found.status !== 404)
210
+ throw await failure(found, `HEAD bucket ${options.bucket}`);
211
+ const region = options.region ?? "us-east-1";
212
+ const body = region === "us-east-1"
213
+ ? undefined
214
+ : `<CreateBucketConfiguration><LocationConstraint>${escapeXml(region)}</LocationConstraint></CreateBucketConfiguration>`;
215
+ const created = await send(bucketUrl, {
216
+ method: "PUT",
217
+ ...(body
218
+ ? {
219
+ body,
220
+ headers: {
221
+ "content-type": "application/xml",
222
+ "x-amz-content-sha256": sha256Hex(new TextEncoder().encode(body)),
223
+ },
224
+ }
225
+ : {}),
226
+ });
227
+ if (created.ok) {
228
+ await created.body?.cancel();
229
+ return;
230
+ }
231
+ const error = await failure(created, `CreateBucket ${options.bucket}`);
232
+ if (error.code === "BucketAlreadyOwnedByYou" || error.code === "BucketAlreadyExists")
233
+ return;
234
+ throw error;
235
+ },
236
+ async put(key, body, put = {}) {
237
+ assertBlobKey(key);
238
+ assertMaxBytes(put.maxBytes);
239
+ put.signal?.throwIfAborted();
240
+ const state = { size: 0, hash: createHash("sha256") };
241
+ const chunks = chunksOf(body)[Symbol.asyncIterator]();
242
+ // Buffer up to one part: a body that ends first goes in one PutObject.
243
+ const buffered = [];
244
+ let length = 0;
245
+ let ended = false;
246
+ try {
247
+ while (length < partSize) {
248
+ const next = await chunks.next();
249
+ if (next.done) {
250
+ ended = true;
251
+ break;
252
+ }
253
+ put.signal?.throwIfAborted();
254
+ state.size += next.value.byteLength;
255
+ if (put.maxBytes !== undefined && state.size > put.maxBytes)
256
+ throw new BlobTooLargeError(put.maxBytes);
257
+ state.hash.update(next.value);
258
+ buffered.push(next.value);
259
+ length += next.value.byteLength;
260
+ }
261
+ const first = concat(buffered, length);
262
+ if (ended) {
263
+ const response = await putBytes(objectUrl(key), first, { "content-type": put.contentType ?? DEFAULT_CONTENT_TYPE }, put.signal);
264
+ if (!response.ok)
265
+ throw await failure(response, `PUT ${key}`);
266
+ await response.body?.cancel();
267
+ }
268
+ else {
269
+ await putMultipart(key, first, chunks, state, put);
270
+ }
271
+ }
272
+ catch (error) {
273
+ await chunks.return?.(undefined).catch(() => undefined);
274
+ throw error;
275
+ }
276
+ return { key, size: state.size, sha256: state.hash.digest("hex") };
277
+ },
278
+ async get(key, get = {}) {
279
+ assertBlobKey(key);
280
+ const range = get.range;
281
+ if (range)
282
+ assertRange(range);
283
+ const response = await send(objectUrl(key), {
284
+ method: "GET",
285
+ ...(range ? { headers: { range: `bytes=${range.start}-${range.end ?? ""}` } } : {}),
286
+ ...(get.signal ? { signal: get.signal } : {}),
287
+ });
288
+ if (response.status === 404) {
289
+ await response.body?.cancel();
290
+ return undefined;
291
+ }
292
+ if (response.status === 416 && range) {
293
+ await response.body?.cancel();
294
+ const total = /\/(\d+)$/.exec(response.headers.get("content-range") ?? "")?.[1];
295
+ throw new BlobRangeError(range, total !== undefined ? Number(total) : ((await head(key))?.size ?? 0));
296
+ }
297
+ if (!response.ok || !response.body)
298
+ throw await failure(response, `GET ${key}`);
299
+ const length = Number(response.headers.get("content-length") ?? 0);
300
+ if (response.status === 206) {
301
+ const match = /^bytes (\d+)-(\d+)\/(\d+|\*)$/.exec(response.headers.get("content-range") ?? "");
302
+ if (!match) {
303
+ await response.body.cancel();
304
+ throw new BlobStoreError(`GET ${key} answered 206 without a Content-Range`);
305
+ }
306
+ const size = match[3] === "*" ? ((await head(key))?.size ?? 0) : Number(match[3]);
307
+ return {
308
+ ...metaOf(key, response.headers, size),
309
+ body: response.body,
310
+ range: { start: Number(match[1]), end: Number(match[2]) },
311
+ };
312
+ }
313
+ // 200: the whole object. A server that ignores Range sends it all too.
314
+ const meta = metaOf(key, response.headers, length);
315
+ if (!range)
316
+ return { ...meta, body: response.body };
317
+ if (range.start >= length) {
318
+ await response.body.cancel();
319
+ throw new BlobRangeError(range, length);
320
+ }
321
+ if (range.start !== 0) {
322
+ await response.body.cancel();
323
+ throw new BlobStoreError(`GET ${key} ignored the Range header`);
324
+ }
325
+ return { ...meta, body: response.body, range: { start: 0, end: length - 1 } };
326
+ },
327
+ head,
328
+ async delete(key) {
329
+ assertBlobKey(key);
330
+ const response = await send(objectUrl(key), { method: "DELETE" });
331
+ if (response.ok || response.status === 404) {
332
+ await response.body?.cancel();
333
+ return;
334
+ }
335
+ throw await failure(response, `DELETE ${key}`);
336
+ },
337
+ async *list(prefix) {
338
+ assertBlobPrefix(prefix);
339
+ let token;
340
+ do {
341
+ const url = new URL(bucketUrl);
342
+ url.searchParams.set("list-type", "2");
343
+ if (prefix !== "")
344
+ url.searchParams.set("prefix", prefix);
345
+ if (token)
346
+ url.searchParams.set("continuation-token", token);
347
+ const response = await send(url.toString(), { method: "GET" });
348
+ if (!response.ok)
349
+ throw await failure(response, `ListObjectsV2 ${prefix}`);
350
+ const xml = await response.text();
351
+ for (const entry of blocks(xml, "Contents")) {
352
+ const key = tag(entry, "Key");
353
+ if (key === undefined)
354
+ continue;
355
+ const modified = tag(entry, "LastModified");
356
+ yield {
357
+ key,
358
+ size: Number(tag(entry, "Size") ?? 0),
359
+ lastModified: modified ? new Date(modified) : new Date(0),
360
+ };
361
+ }
362
+ token = tag(xml, "IsTruncated") === "true" ? tag(xml, "NextContinuationToken") : undefined;
363
+ } while (token);
364
+ },
365
+ };
366
+ }