@elaraai/e3-api-server 1.0.79 → 1.0.80
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +83 -19
- package/dist/src/beast2.js +1 -1
- package/dist/src/beast2.js.map +1 -1
- package/dist/src/cli.js +13 -0
- package/dist/src/cli.js.map +1 -1
- package/dist/src/errors.d.ts.map +1 -1
- package/dist/src/errors.js +8 -1
- package/dist/src/errors.js.map +1 -1
- package/dist/src/handlers/dataflow.d.ts +58 -11
- package/dist/src/handlers/dataflow.d.ts.map +1 -1
- package/dist/src/handlers/dataflow.js +120 -48
- package/dist/src/handlers/dataflow.js.map +1 -1
- package/dist/src/handlers/dataflow.spec.d.ts +6 -0
- package/dist/src/handlers/dataflow.spec.d.ts.map +1 -0
- package/dist/src/handlers/dataflow.spec.js +106 -0
- package/dist/src/handlers/dataflow.spec.js.map +1 -0
- package/dist/src/handlers/datasets.d.ts +60 -34
- package/dist/src/handlers/datasets.d.ts.map +1 -1
- package/dist/src/handlers/datasets.js +452 -306
- package/dist/src/handlers/datasets.js.map +1 -1
- package/dist/src/handlers/datasets.spec.js +338 -119
- package/dist/src/handlers/datasets.spec.js.map +1 -1
- package/dist/src/handlers/functions.d.ts.map +1 -1
- package/dist/src/handlers/functions.js +10 -8
- package/dist/src/handlers/functions.js.map +1 -1
- package/dist/src/handlers/functions.spec.js +2 -2
- package/dist/src/handlers/functions.spec.js.map +1 -1
- package/dist/src/handlers/index.d.ts +2 -2
- package/dist/src/handlers/index.d.ts.map +1 -1
- package/dist/src/handlers/index.js +2 -2
- package/dist/src/handlers/index.js.map +1 -1
- package/dist/src/handlers/records.d.ts.map +1 -1
- package/dist/src/handlers/records.js +15 -12
- package/dist/src/handlers/records.js.map +1 -1
- package/dist/src/handlers/records.spec.js +9 -5
- package/dist/src/handlers/records.spec.js.map +1 -1
- package/dist/src/handlers/repository.d.ts +59 -10
- package/dist/src/handlers/repository.d.ts.map +1 -1
- package/dist/src/handlers/repository.js +147 -40
- package/dist/src/handlers/repository.js.map +1 -1
- package/dist/src/handlers/tasks.d.ts +2 -1
- package/dist/src/handlers/tasks.d.ts.map +1 -1
- package/dist/src/handlers/tasks.js +31 -8
- package/dist/src/handlers/tasks.js.map +1 -1
- package/dist/src/handlers/tasks.spec.d.ts +6 -0
- package/dist/src/handlers/tasks.spec.d.ts.map +1 -0
- package/dist/src/handlers/tasks.spec.js +64 -0
- package/dist/src/handlers/tasks.spec.js.map +1 -0
- package/dist/src/handlers/workspaces.d.ts +44 -6
- package/dist/src/handlers/workspaces.d.ts.map +1 -1
- package/dist/src/handlers/workspaces.js +74 -40
- package/dist/src/handlers/workspaces.js.map +1 -1
- package/dist/src/handlers/workspaces.spec.d.ts +6 -0
- package/dist/src/handlers/workspaces.spec.d.ts.map +1 -0
- package/dist/src/handlers/workspaces.spec.js +164 -0
- package/dist/src/handlers/workspaces.spec.js.map +1 -0
- package/dist/src/index.d.ts +3 -0
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +7 -0
- package/dist/src/index.js.map +1 -1
- package/dist/src/local-dataflow.d.ts +14 -0
- package/dist/src/local-dataflow.d.ts.map +1 -0
- package/dist/src/local-dataflow.js +41 -0
- package/dist/src/local-dataflow.js.map +1 -0
- package/dist/src/middleware/repository.d.ts +39 -0
- package/dist/src/middleware/repository.d.ts.map +1 -0
- package/dist/src/middleware/repository.js +77 -0
- package/dist/src/middleware/repository.js.map +1 -0
- package/dist/src/routes/data.d.ts +2 -2
- package/dist/src/routes/data.d.ts.map +1 -1
- package/dist/src/routes/data.js +22 -45
- package/dist/src/routes/data.js.map +1 -1
- package/dist/src/routes/datasets.d.ts +0 -3
- package/dist/src/routes/datasets.d.ts.map +1 -1
- package/dist/src/routes/datasets.js +40 -8
- package/dist/src/routes/datasets.js.map +1 -1
- package/dist/src/routes/executions.d.ts +37 -2
- package/dist/src/routes/executions.d.ts.map +1 -1
- package/dist/src/routes/executions.js +21 -12
- package/dist/src/routes/executions.js.map +1 -1
- package/dist/src/routes/index.d.ts +2 -1
- package/dist/src/routes/index.d.ts.map +1 -1
- package/dist/src/routes/index.js +1 -0
- package/dist/src/routes/index.js.map +1 -1
- package/dist/src/routes/objects.d.ts +16 -2
- package/dist/src/routes/objects.d.ts.map +1 -1
- package/dist/src/routes/objects.js +34 -3
- package/dist/src/routes/objects.js.map +1 -1
- package/dist/src/routes/repositories.d.ts +26 -0
- package/dist/src/routes/repositories.d.ts.map +1 -0
- package/dist/src/routes/repositories.js +58 -0
- package/dist/src/routes/repositories.js.map +1 -0
- package/dist/src/routes/repositories.spec.d.ts +6 -0
- package/dist/src/routes/repositories.spec.d.ts.map +1 -0
- package/dist/src/routes/repositories.spec.js +77 -0
- package/dist/src/routes/repositories.spec.js.map +1 -0
- package/dist/src/routes/repository.d.ts +12 -2
- package/dist/src/routes/repository.d.ts.map +1 -1
- package/dist/src/routes/repository.js +24 -9
- package/dist/src/routes/repository.js.map +1 -1
- package/dist/src/routes/transfer.d.ts +11 -9
- package/dist/src/routes/transfer.d.ts.map +1 -1
- package/dist/src/routes/transfer.js +90 -115
- package/dist/src/routes/transfer.js.map +1 -1
- package/dist/src/routes/workspaces.d.ts +14 -1
- package/dist/src/routes/workspaces.d.ts.map +1 -1
- package/dist/src/routes/workspaces.js +57 -46
- package/dist/src/routes/workspaces.js.map +1 -1
- package/dist/src/server.d.ts +18 -6
- package/dist/src/server.d.ts.map +1 -1
- package/dist/src/server.js +55 -182
- package/dist/src/server.js.map +1 -1
- package/dist/src/server.spec.d.ts +6 -0
- package/dist/src/server.spec.d.ts.map +1 -0
- package/dist/src/server.spec.js +59 -0
- package/dist/src/server.spec.js.map +1 -0
- package/dist/src/types.d.ts +200 -16
- package/dist/src/types.d.ts.map +1 -1
- package/dist/src/types.js +5 -2
- package/dist/src/types.js.map +1 -1
- package/package.json +5 -5
- package/dist/src/async-operation-state.d.ts +0 -37
- package/dist/src/async-operation-state.d.ts.map +0 -1
- package/dist/src/async-operation-state.js +0 -118
- package/dist/src/async-operation-state.js.map +0 -1
- package/dist/src/orchestrator-manager.d.ts +0 -45
- package/dist/src/orchestrator-manager.d.ts.map +0 -1
- package/dist/src/orchestrator-manager.js +0 -150
- package/dist/src/orchestrator-manager.js.map +0 -1
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Copyright (c) 2025 Elara AI Pty Ltd
|
|
3
|
+
* Licensed under BSL 1.1. See LICENSE for details.
|
|
4
|
+
*/
|
|
5
|
+
import { repositoryOpen } from '@elaraai/e3-core';
|
|
6
|
+
import { sendJsonError } from '../errors.js';
|
|
7
|
+
/** Whether a request creates or removes a repository. */
|
|
8
|
+
function createsOrRemoves(c) {
|
|
9
|
+
return (c.req.method === 'PUT' || c.req.method === 'DELETE') && /\/repos\/[^/]+$/.test(c.req.path);
|
|
10
|
+
}
|
|
11
|
+
/**
|
|
12
|
+
* The gate of a host's repositories: the repository exists, is not being
|
|
13
|
+
* removed, and is opened as every way into a repository opens it — upgraded in
|
|
14
|
+
* place when an older release wrote it, and refused, naming the fix, when this
|
|
15
|
+
* e3 cannot read it. A repository being removed answers its status alone, and
|
|
16
|
+
* is not opened. It goes through the storage backend's `RepoStore`, so a host
|
|
17
|
+
* mounts it over its own.
|
|
18
|
+
*
|
|
19
|
+
* @param storage - Storage backend, whose `RepoStore` keeps the repositories
|
|
20
|
+
* @param getRepoPath - A repository's identifier from its name, as the stores
|
|
21
|
+
* other than the `RepoStore` take it
|
|
22
|
+
* @returns The middleware
|
|
23
|
+
*/
|
|
24
|
+
export function createRepositoryGate(storage, getRepoPath) {
|
|
25
|
+
return async (c, next) => {
|
|
26
|
+
if (createsOrRemoves(c)) {
|
|
27
|
+
await next();
|
|
28
|
+
return;
|
|
29
|
+
}
|
|
30
|
+
const repo = c.req.param('repo');
|
|
31
|
+
// A repository whose metadata this e3 cannot read is refused naming the
|
|
32
|
+
// fix, not with a bare 500.
|
|
33
|
+
let metadata;
|
|
34
|
+
try {
|
|
35
|
+
metadata = await storage.repos.getMetadata(repo);
|
|
36
|
+
}
|
|
37
|
+
catch (err) {
|
|
38
|
+
return sendJsonError(err);
|
|
39
|
+
}
|
|
40
|
+
if (!metadata) {
|
|
41
|
+
return c.json({ error: 'not_found', message: `Repository '${repo}' not found` }, 404);
|
|
42
|
+
}
|
|
43
|
+
if (metadata.status.type === 'deleting') {
|
|
44
|
+
if (!/\/repos\/[^/]+\/status$/.test(c.req.path)) {
|
|
45
|
+
return c.json({ error: 'not_found', message: `Repository '${repo}' not found` }, 404);
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
else {
|
|
49
|
+
try {
|
|
50
|
+
await repositoryOpen(storage, getRepoPath(repo));
|
|
51
|
+
}
|
|
52
|
+
catch (err) {
|
|
53
|
+
return sendJsonError(err);
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
await next();
|
|
57
|
+
};
|
|
58
|
+
}
|
|
59
|
+
/**
|
|
60
|
+
* The gate of a server of one repository: the repository is `default`, and a
|
|
61
|
+
* request to any other is not found. The server opens its repository as it
|
|
62
|
+
* starts.
|
|
63
|
+
*
|
|
64
|
+
* @returns The middleware
|
|
65
|
+
*/
|
|
66
|
+
export function createSingleRepositoryGate() {
|
|
67
|
+
return async (c, next) => {
|
|
68
|
+
if (!createsOrRemoves(c)) {
|
|
69
|
+
const repo = c.req.param('repo');
|
|
70
|
+
if (repo !== 'default') {
|
|
71
|
+
return c.json({ error: 'not_found', message: `Repository '${repo}' not found` }, 404);
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
await next();
|
|
75
|
+
};
|
|
76
|
+
}
|
|
77
|
+
//# sourceMappingURL=repository.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"repository.js","sourceRoot":"","sources":["../../../src/middleware/repository.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAcH,OAAO,EAAE,cAAc,EAAuB,MAAM,kBAAkB,CAAC;AACvE,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAE7C,yDAAyD;AACzD,SAAS,gBAAgB,CAAC,CAAU;IAClC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,KAAK,KAAK,IAAI,CAAC,CAAC,GAAG,CAAC,MAAM,KAAK,QAAQ,CAAC,IAAI,iBAAiB,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;AACrG,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,UAAU,oBAAoB,CAClC,OAAuB,EACvB,WAAqC;IAErC,OAAO,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,EAAE;QACvB,IAAI,gBAAgB,CAAC,CAAC,CAAC,EAAE,CAAC;YACxB,MAAM,IAAI,EAAE,CAAC;YACb,OAAO;QACT,CAAC;QAED,MAAM,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAE,CAAC;QAElC,wEAAwE;QACxE,4BAA4B;QAC5B,IAAI,QAAQ,CAAC;QACb,IAAI,CAAC;YACH,QAAQ,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;QACnD,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO,aAAa,CAAC,GAAG,CAAC,CAAC;QAC5B,CAAC;QACD,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,OAAO,CAAC,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,WAAW,EAAE,OAAO,EAAE,eAAe,IAAI,aAAa,EAAE,EAAE,GAAG,CAAC,CAAC;QACxF,CAAC;QAED,IAAI,QAAQ,CAAC,MAAM,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;YACxC,IAAI,CAAC,yBAAyB,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;gBAChD,OAAO,CAAC,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,WAAW,EAAE,OAAO,EAAE,eAAe,IAAI,aAAa,EAAE,EAAE,GAAG,CAAC,CAAC;YACxF,CAAC;QACH,CAAC;aAAM,CAAC;YACN,IAAI,CAAC;gBACH,MAAM,cAAc,CAAC,OAAO,EAAE,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC;YACnD,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,OAAO,aAAa,CAAC,GAAG,CAAC,CAAC;YAC5B,CAAC;QACH,CAAC;QAED,MAAM,IAAI,EAAE,CAAC;IACf,CAAC,CAAC;AACJ,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,0BAA0B;IACxC,OAAO,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,EAAE;QACvB,IAAI,CAAC,gBAAgB,CAAC,CAAC,CAAC,EAAE,CAAC;YACzB,MAAM,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;YACjC,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;gBACvB,OAAO,CAAC,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,WAAW,EAAE,OAAO,EAAE,eAAe,IAAI,aAAa,EAAE,EAAE,GAAG,CAAC,CAAC;YACxF,CAAC;QACH,CAAC;QACD,MAAM,IAAI,EAAE,CAAC;IACf,CAAC,CAAC;AACJ,CAAC"}
|
|
@@ -8,8 +8,8 @@ import { type StorageBackend, type TransferBackend } from '@elaraai/e3-core';
|
|
|
8
8
|
* Create generic upload/download data endpoints.
|
|
9
9
|
*
|
|
10
10
|
* Returns two Hono apps:
|
|
11
|
-
* - `uploads`: PUT /:id — upload
|
|
12
|
-
* PUT /:id/parts/:part — one part of a dataset upload
|
|
11
|
+
* - `uploads`: PUT /:id — upload a package zip;
|
|
12
|
+
* PUT /:id/parts/:part — one part of a dataset upload
|
|
13
13
|
* - `downloads`: GET /:id — download data (dataset BEAST2 or package zip)
|
|
14
14
|
*
|
|
15
15
|
* These are unauthenticated — the UUID in the URL is the sole capability.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../src/routes/data.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../src/routes/data.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAO5B,OAAO,EAKL,KAAK,cAAc,EACnB,KAAK,eAAe,EACrB,MAAM,kBAAkB,CAAC;AAiB1B;;;;;;;;;;GAUG;AACH,wBAAgB,mBAAmB,CACjC,eAAe,EAAE,eAAe,EAChC,OAAO,EAAE,cAAc,EACvB,WAAW,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,MAAM;;;EAgKtC"}
|
package/dist/src/routes/data.js
CHANGED
|
@@ -7,12 +7,9 @@ import { mkdir, open, writeFile, readFile, unlink } from 'node:fs/promises';
|
|
|
7
7
|
import { createWriteStream } from 'node:fs';
|
|
8
8
|
import { Readable, Transform } from 'node:stream';
|
|
9
9
|
import { pipeline } from 'node:stream/promises';
|
|
10
|
-
import { join } from 'node:path';
|
|
11
|
-
import { tmpdir } from 'node:os';
|
|
12
10
|
import { variant } from '@elaraai/east';
|
|
13
|
-
import { transferPartCount, transferPartRange } from '@elaraai/e3-types';
|
|
14
|
-
import {
|
|
15
|
-
const STAGING_DIR = join(tmpdir(), 'e3-transfers');
|
|
11
|
+
import { BEAST2_CONTENT_TYPE, transferPartCount, transferPartRange } from '@elaraai/e3-types';
|
|
12
|
+
import { ObjectNotFoundError, packageStagingPath, transferStagingDir, transferStagingPath, } from '@elaraai/e3-core';
|
|
16
13
|
/**
|
|
17
14
|
* Reject requests that carry an Authorization header.
|
|
18
15
|
*
|
|
@@ -31,8 +28,8 @@ function rejectAuthHeader(c) {
|
|
|
31
28
|
* Create generic upload/download data endpoints.
|
|
32
29
|
*
|
|
33
30
|
* Returns two Hono apps:
|
|
34
|
-
* - `uploads`: PUT /:id — upload
|
|
35
|
-
* PUT /:id/parts/:part — one part of a dataset upload
|
|
31
|
+
* - `uploads`: PUT /:id — upload a package zip;
|
|
32
|
+
* PUT /:id/parts/:part — one part of a dataset upload
|
|
36
33
|
* - `downloads`: GET /:id — download data (dataset BEAST2 or package zip)
|
|
37
34
|
*
|
|
38
35
|
* These are unauthenticated — the UUID in the URL is the sole capability.
|
|
@@ -41,40 +38,13 @@ function rejectAuthHeader(c) {
|
|
|
41
38
|
export function createDataEndpoints(transferBackend, storage, getRepoPath) {
|
|
42
39
|
const uploads = new Hono();
|
|
43
40
|
const downloads = new Hono();
|
|
44
|
-
// PUT /api/uploads/:id — Upload
|
|
41
|
+
// PUT /api/uploads/:id — Upload a package zip, staged for the trigger
|
|
42
|
+
// endpoint to process
|
|
45
43
|
uploads.put('/:id', async (c) => {
|
|
46
44
|
const rejected = rejectAuthHeader(c);
|
|
47
45
|
if (rejected)
|
|
48
46
|
return rejected;
|
|
49
47
|
const id = c.req.param('id');
|
|
50
|
-
// Try dataset upload first
|
|
51
|
-
const dsRecord = await transferBackend.datasetUpload.get(id);
|
|
52
|
-
if (dsRecord) {
|
|
53
|
-
// Streamed to the repo's own staging area: a delivery can be far larger
|
|
54
|
-
// than this process's heap, and the commit turns the staged file into an
|
|
55
|
-
// object by link or rename — which only works on the repo's device.
|
|
56
|
-
const repoPath = getRepoPath(dsRecord.repo);
|
|
57
|
-
const stagingPath = transferStagingPath(repoPath, id);
|
|
58
|
-
await mkdir(transferStagingDir(repoPath), { recursive: true });
|
|
59
|
-
const stream = c.req.raw.body;
|
|
60
|
-
try {
|
|
61
|
-
if (stream) {
|
|
62
|
-
await pipeline(Readable.fromWeb(stream), createWriteStream(stagingPath));
|
|
63
|
-
}
|
|
64
|
-
else {
|
|
65
|
-
await writeFile(stagingPath, new Uint8Array(await c.req.arrayBuffer()));
|
|
66
|
-
}
|
|
67
|
-
}
|
|
68
|
-
catch (err) {
|
|
69
|
-
// A broken upload leaves no partial file behind; gc sweeps the staging
|
|
70
|
-
// directory only as the backstop for a server that crashed mid-upload.
|
|
71
|
-
// The transfer record stays, so the client may retry the PUT.
|
|
72
|
-
await unlink(stagingPath).catch(() => { });
|
|
73
|
-
throw err;
|
|
74
|
-
}
|
|
75
|
-
return new Response(null, { status: 200 });
|
|
76
|
-
}
|
|
77
|
-
// Try package import — stage file for later processing by trigger endpoint
|
|
78
48
|
const pkgRecord = await transferBackend.packageImport.get(id);
|
|
79
49
|
if (pkgRecord) {
|
|
80
50
|
const body = new Uint8Array(await c.req.arrayBuffer());
|
|
@@ -82,16 +52,18 @@ export function createDataEndpoints(transferBackend, storage, getRepoPath) {
|
|
|
82
52
|
await transferBackend.packageImport.delete(id);
|
|
83
53
|
return new Response(`size mismatch: expected ${pkgRecord.size}, got ${body.byteLength}`, { status: 400 });
|
|
84
54
|
}
|
|
85
|
-
const
|
|
86
|
-
await mkdir(
|
|
87
|
-
await writeFile(
|
|
55
|
+
const repoPath = getRepoPath(pkgRecord.repo);
|
|
56
|
+
await mkdir(transferStagingDir(repoPath), { recursive: true });
|
|
57
|
+
await writeFile(packageStagingPath(repoPath, id), body);
|
|
88
58
|
await transferBackend.packageImport.updateStatus(id, variant('uploaded', null));
|
|
89
59
|
return new Response(null, { status: 200 });
|
|
90
60
|
}
|
|
91
61
|
return new Response('Not found', { status: 404 });
|
|
92
62
|
});
|
|
93
|
-
// PUT /api/uploads/:id/parts/:part — One part of a dataset upload
|
|
94
|
-
//
|
|
63
|
+
// PUT /api/uploads/:id/parts/:part — One part of a dataset upload. The staged
|
|
64
|
+
// file sits in the repository, far larger than this process's heap if need
|
|
65
|
+
// be, because the commit turns it into an object by link or rename, which
|
|
66
|
+
// only works on the repository's device.
|
|
95
67
|
uploads.put('/:id/parts/:part', async (c) => {
|
|
96
68
|
const rejected = rejectAuthHeader(c);
|
|
97
69
|
if (rejected)
|
|
@@ -102,6 +74,11 @@ export function createDataEndpoints(transferBackend, storage, getRepoPath) {
|
|
|
102
74
|
if (!record || partBytes === null) {
|
|
103
75
|
return new Response('Not found', { status: 404 });
|
|
104
76
|
}
|
|
77
|
+
// A part sent once the commit is under way could rewrite the bytes it
|
|
78
|
+
// verifies.
|
|
79
|
+
if ((await transferBackend.datasetUpload.getCommitStatus(id)) !== null) {
|
|
80
|
+
return new Response('the upload is committed: it takes no more parts', { status: 409 });
|
|
81
|
+
}
|
|
105
82
|
const part = Number(c.req.param('part'));
|
|
106
83
|
const range = transferPartRange(record.size, partBytes, part);
|
|
107
84
|
if (!range) {
|
|
@@ -109,9 +86,9 @@ export function createDataEndpoints(transferBackend, storage, getRepoPath) {
|
|
|
109
86
|
}
|
|
110
87
|
// Every part streams to its own offset in the one staged file — in any
|
|
111
88
|
// order, concurrently, and a re-sent part over the old one — so the commit
|
|
112
|
-
// adopts the file
|
|
113
|
-
//
|
|
114
|
-
//
|
|
89
|
+
// adopts the file with nothing to assemble. Opening with 'a' creates the
|
|
90
|
+
// file without truncating the parts already there; the write itself needs
|
|
91
|
+
// a positioned handle.
|
|
115
92
|
const repoPath = getRepoPath(record.repo);
|
|
116
93
|
const stagingPath = transferStagingPath(repoPath, id);
|
|
117
94
|
await mkdir(transferStagingDir(repoPath), { recursive: true });
|
|
@@ -178,7 +155,7 @@ export function createDataEndpoints(transferBackend, storage, getRepoPath) {
|
|
|
178
155
|
// Try package export
|
|
179
156
|
const pkgRecord = await transferBackend.packageExport.get(id);
|
|
180
157
|
if (pkgRecord && pkgRecord.status.type === 'completed') {
|
|
181
|
-
const stagingPath =
|
|
158
|
+
const stagingPath = packageStagingPath(getRepoPath(pkgRecord.repo), id);
|
|
182
159
|
try {
|
|
183
160
|
const fileData = await readFile(stagingPath);
|
|
184
161
|
await unlink(stagingPath).catch(() => { });
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"data.js","sourceRoot":"","sources":["../../../src/routes/data.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAC5B,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC5E,OAAO,EAAE,iBAAiB,EAAE,MAAM,SAAS,CAAC;AAC5C,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAClD,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAChD,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"data.js","sourceRoot":"","sources":["../../../src/routes/data.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAC5B,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC5E,OAAO,EAAE,iBAAiB,EAAE,MAAM,SAAS,CAAC;AAC5C,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAClD,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAChD,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AACxC,OAAO,EAAE,mBAAmB,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAC9F,OAAO,EACL,mBAAmB,EACnB,kBAAkB,EAClB,kBAAkB,EAClB,mBAAmB,GAGpB,MAAM,kBAAkB,CAAC;AAE1B;;;;;;;GAOG;AACH,SAAS,gBAAgB,CAAC,CAAwD;IAChF,IAAI,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,eAAe,CAAC,EAAE,CAAC;QAClC,OAAO,IAAI,QAAQ,CAAC,yDAAyD,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC;IAClG,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,UAAU,mBAAmB,CACjC,eAAgC,EAChC,OAAuB,EACvB,WAAqC;IAErC,MAAM,OAAO,GAAG,IAAI,IAAI,EAAE,CAAC;IAC3B,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC;IAE7B,sEAAsE;IACtE,sBAAsB;IACtB,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;QAC9B,MAAM,QAAQ,GAAG,gBAAgB,CAAC,CAAC,CAAC,CAAC;QACrC,IAAI,QAAQ;YAAE,OAAO,QAAQ,CAAC;QAE9B,MAAM,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAE,CAAC;QAC9B,MAAM,SAAS,GAAG,MAAM,eAAe,CAAC,aAAa,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC9D,IAAI,SAAS,EAAE,CAAC;YACd,MAAM,IAAI,GAAG,IAAI,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC,CAAC;YACvD,IAAI,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,SAAS,CAAC,IAAI,EAAE,CAAC;gBAC/C,MAAM,eAAe,CAAC,aAAa,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;gBAC/C,OAAO,IAAI,QAAQ,CACjB,2BAA2B,SAAS,CAAC,IAAI,SAAS,IAAI,CAAC,UAAU,EAAE,EACnE,EAAE,MAAM,EAAE,GAAG,EAAE,CAChB,CAAC;YACJ,CAAC;YAED,MAAM,QAAQ,GAAG,WAAW,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;YAC7C,MAAM,KAAK,CAAC,kBAAkB,CAAC,QAAQ,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YAC/D,MAAM,SAAS,CAAC,kBAAkB,CAAC,QAAQ,EAAE,EAAE,CAAC,EAAE,IAAI,CAAC,CAAC;YACxD,MAAM,eAAe,CAAC,aAAa,CAAC,YAAY,CAAC,EAAE,EAAE,OAAO,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC,CAAC;YAEhF,OAAO,IAAI,QAAQ,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC;QAC7C,CAAC;QAED,OAAO,IAAI,QAAQ,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC;IACpD,CAAC,CAAC,CAAC;IAEH,8EAA8E;IAC9E,2EAA2E;IAC3E,0EAA0E;IAC1E,yCAAyC;IACzC,OAAO,CAAC,GAAG,CAAC,kBAAkB,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;QAC1C,MAAM,QAAQ,GAAG,gBAAgB,CAAC,CAAC,CAAC,CAAC;QACrC,IAAI,QAAQ;YAAE,OAAO,QAAQ,CAAC;QAE9B,MAAM,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAE,CAAC;QAC9B,MAAM,MAAM,GAAG,MAAM,eAAe,CAAC,aAAa,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC3D,MAAM,SAAS,GAAG,MAAM,CAAC,CAAC,CAAC,MAAM,eAAe,CAAC,aAAa,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QACvF,IAAI,CAAC,MAAM,IAAI,SAAS,KAAK,IAAI,EAAE,CAAC;YAClC,OAAO,IAAI,QAAQ,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC;QACpD,CAAC;QACD,sEAAsE;QACtE,YAAY;QACZ,IAAI,CAAC,MAAM,eAAe,CAAC,aAAa,CAAC,eAAe,CAAC,EAAE,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YACvE,OAAO,IAAI,QAAQ,CAAC,iDAAiD,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC;QAC1F,CAAC;QACD,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;QACzC,MAAM,KAAK,GAAG,iBAAiB,CAAC,MAAM,CAAC,IAAI,EAAE,SAAS,EAAE,IAAI,CAAC,CAAC;QAC9D,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,OAAO,IAAI,QAAQ,CACjB,WAAW,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,oBAAoB,iBAAiB,CAAC,MAAM,CAAC,IAAI,EAAE,SAAS,CAAC,QAAQ,EACnG,EAAE,MAAM,EAAE,GAAG,EAAE,CAChB,CAAC;QACJ,CAAC;QAED,uEAAuE;QACvE,2EAA2E;QAC3E,yEAAyE;QACzE,0EAA0E;QAC1E,uBAAuB;QACvB,MAAM,QAAQ,GAAG,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAC1C,MAAM,WAAW,GAAG,mBAAmB,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;QACtD,MAAM,KAAK,CAAC,kBAAkB,CAAC,QAAQ,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC/D,MAAM,CAAC,MAAM,IAAI,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC;QAE7C,qEAAqE;QACrE,sEAAsE;QACtE,MAAM,QAAQ,GAAG,KAAK,CAAC,GAAG,GAAG,KAAK,CAAC,KAAK,CAAC;QACzC,IAAI,QAAQ,GAAG,CAAC,CAAC;QACjB,MAAM,OAAO,GAAG,IAAI,SAAS,CAAC;YAC5B,SAAS,CAAC,KAAa,EAAE,SAAS,EAAE,QAAQ;gBAC1C,QAAQ,IAAI,KAAK,CAAC,MAAM,CAAC;gBACzB,IAAI,QAAQ,GAAG,QAAQ,EAAE,CAAC;oBACxB,QAAQ,CAAC,IAAI,KAAK,CAAC,QAAQ,IAAI,OAAO,QAAQ,4BAA4B,CAAC,CAAC,CAAC;gBAC/E,CAAC;qBAAM,CAAC;oBACN,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;gBACxB,CAAC;YACH,CAAC;SACF,CAAC,CAAC;QACH,MAAM,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC;QAC5B,IAAI,CAAC;YACH,MAAM,QAAQ,CACZ,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,IAA8C,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC,EAC3F,OAAO,EACP,iBAAiB,CAAC,WAAW,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC,CACpE,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,QAAQ,GAAG,QAAQ,EAAE,CAAC;gBACxB,OAAO,IAAI,QAAQ,CAAC,QAAQ,IAAI,OAAO,QAAQ,4BAA4B,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC;YAChG,CAAC;YACD,MAAM,GAAG,CAAC;QACZ,CAAC;QACD,IAAI,QAAQ,KAAK,QAAQ,EAAE,CAAC;YAC1B,OAAO,IAAI,QAAQ,CAAC,QAAQ,IAAI,OAAO,QAAQ,eAAe,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC;QAC7F,CAAC;QACD,OAAO,IAAI,QAAQ,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC;IAC7C,CAAC,CAAC,CAAC;IAEH,yEAAyE;IACzE,SAAS,CAAC,GAAG,CAAC,MAAM,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;QAChC,MAAM,QAAQ,GAAG,gBAAgB,CAAC,CAAC,CAAC,CAAC;QACrC,IAAI,QAAQ;YAAE,OAAO,QAAQ,CAAC;QAE9B,MAAM,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAE,CAAC;QAE9B,uBAAuB;QACvB,MAAM,QAAQ,GAAG,MAAM,eAAe,CAAC,eAAe,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC/D,IAAI,QAAQ,EAAE,CAAC;YACb,MAAM,QAAQ,GAAG,WAAW,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;YAC5C,IAAI,CAAC;gBACH,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC;gBACjE,MAAM,eAAe,CAAC,eAAe,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;gBACjD,OAAO,IAAI,QAAQ,CAAC,IAAI,EAAE;oBACxB,OAAO,EAAE;wBACP,cAAc,EAAE,mBAAmB;wBACnC,gBAAgB,EAAE,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC;wBACzC,kBAAkB,EAAE,QAAQ,CAAC,IAAI;qBAClC;iBACF,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,eAAe,CAAC,eAAe,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;gBACjD,IAAI,GAAG,YAAY,mBAAmB,EAAE,CAAC;oBACvC,OAAO,IAAI,QAAQ,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC;gBACpD,CAAC;gBACD,MAAM,GAAG,CAAC;YACZ,CAAC;QACH,CAAC;QAED,qBAAqB;QACrB,MAAM,SAAS,GAAG,MAAM,eAAe,CAAC,aAAa,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC9D,IAAI,SAAS,IAAI,SAAS,CAAC,MAAM,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;YACvD,MAAM,WAAW,GAAG,kBAAkB,CAAC,WAAW,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;YACxE,IAAI,CAAC;gBACH,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAC,WAAW,CAAC,CAAC;gBAC7C,MAAM,MAAM,CAAC,WAAW,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;gBAC1C,MAAM,eAAe,CAAC,aAAa,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;gBAE/C,OAAO,IAAI,QAAQ,CAAC,QAAQ,EAAE;oBAC5B,MAAM,EAAE,GAAG;oBACX,OAAO,EAAE;wBACP,cAAc,EAAE,iBAAiB;wBACjC,gBAAgB,EAAE,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC;qBAC9C;iBACF,CAAC,CAAC;YACL,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO,IAAI,QAAQ,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC;YACpD,CAAC;QACH,CAAC;QAED,OAAO,IAAI,QAAQ,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC;IACpD,CAAC,CAAC,CAAC;IAEH,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC;AAChC,CAAC"}
|
|
@@ -10,9 +10,6 @@ export interface DatasetRouteOptions {
|
|
|
10
10
|
* 4 MiB; deployments with tighter response limits (e.g. Lambda proxy's
|
|
11
11
|
* 6 MB, base64-inflated) pass a smaller budget. */
|
|
12
12
|
pageByteBudget?: number;
|
|
13
|
-
/** Absolute blob-buffering cap for the page endpoint. Defaults to
|
|
14
|
-
* 512 MiB, until range reads land. */
|
|
15
|
-
pageReadMaxBytes?: number;
|
|
16
13
|
}
|
|
17
14
|
export declare function createDatasetRoutes(storage: StorageBackend, getRepoPath: (repo: string) => string, transferBackend?: TransferBackend, options?: DatasetRouteOptions): Hono<import("hono/types").BlankEnv, import("hono/types").BlankSchema, "/">;
|
|
18
15
|
//# sourceMappingURL=datasets.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"datasets.d.ts","sourceRoot":"","sources":["../../../src/routes/datasets.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAE5B,OAAO,KAAK,EAAE,cAAc,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAaxE,+CAA+C;AAC/C,MAAM,WAAW,mBAAmB;IAClC;;wDAEoD;IACpD,cAAc,CAAC,EAAE,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"datasets.d.ts","sourceRoot":"","sources":["../../../src/routes/datasets.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAE5B,OAAO,KAAK,EAAE,cAAc,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAaxE,+CAA+C;AAC/C,MAAM,WAAW,mBAAmB;IAClC;;wDAEoD;IACpD,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AASD,wBAAgB,mBAAmB,CACjC,OAAO,EAAE,cAAc,EACvB,WAAW,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,MAAM,EACrC,eAAe,CAAC,EAAE,eAAe,EACjC,OAAO,CAAC,EAAE,mBAAmB,8EA6G9B"}
|
|
@@ -66,13 +66,19 @@ export function createDatasetRoutes(storage, getRepoPath, transferBackend, optio
|
|
|
66
66
|
const key = c.req.query('key');
|
|
67
67
|
const prefix = c.req.query('prefix');
|
|
68
68
|
const fields = c.req.queries('field');
|
|
69
|
+
// `from` / `to` repeat, as `field` does: each is one `.east` literal of
|
|
70
|
+
// a leading prefix of the flattened key.
|
|
71
|
+
const from = c.req.queries('from');
|
|
72
|
+
const to = c.req.queries('to');
|
|
73
|
+
const index = c.req.query('index');
|
|
69
74
|
return findDatasetKey(storage, repoPath, ws, treePath, {
|
|
70
75
|
...(key !== undefined && { key }),
|
|
71
76
|
...(prefix !== undefined && { prefix }),
|
|
72
77
|
...(fields !== undefined && fields.length > 0 && { fields }),
|
|
78
|
+
...(from !== undefined && from.length > 0 && { from }),
|
|
79
|
+
...(to !== undefined && to.length > 0 && { to }),
|
|
80
|
+
...(index !== undefined && index !== '' && { index }),
|
|
73
81
|
...(hash !== undefined && hash !== '' && { hash }),
|
|
74
|
-
}, {
|
|
75
|
-
...(options?.pageReadMaxBytes !== undefined && { readMaxBytes: options.pageReadMaxBytes }),
|
|
76
82
|
});
|
|
77
83
|
}
|
|
78
84
|
if (page) {
|
|
@@ -81,14 +87,17 @@ export function createDatasetRoutes(storage, getRepoPath, transferBackend, optio
|
|
|
81
87
|
...(intParam(c.req.query('offset')) !== undefined && { offset: intParam(c.req.query('offset')) }),
|
|
82
88
|
...(intParam(c.req.query('limit')) !== undefined && { limit: intParam(c.req.query('limit')) }),
|
|
83
89
|
...(intParam(c.req.query('segment')) !== undefined && { segment: intParam(c.req.query('segment')) }),
|
|
90
|
+
...(c.req.query('index') !== undefined && c.req.query('index') !== '' && { index: c.req.query('index') }),
|
|
91
|
+
...(c.req.query('join') === 'true' && { join: true }),
|
|
84
92
|
...(hash !== undefined && hash !== '' && { hash }),
|
|
85
93
|
};
|
|
86
94
|
return getDatasetPage(storage, repoPath, ws, treePath, window, {
|
|
87
95
|
...(options?.pageByteBudget !== undefined && { byteBudget: options.pageByteBudget }),
|
|
88
|
-
...(options?.pageReadMaxBytes !== undefined && { readMaxBytes: options.pageReadMaxBytes }),
|
|
89
96
|
});
|
|
90
97
|
}
|
|
91
|
-
|
|
98
|
+
// `segments=true`: a collection held as a segment manifest is answered
|
|
99
|
+
// with its manifest's hash, for a client that downloads the segments.
|
|
100
|
+
return getDataset(storage, repoPath, ws, treePath, repo, c.req.url, transferBackend, c.req.query('segments') === 'true');
|
|
92
101
|
});
|
|
93
102
|
// PUT /api/repos/:repo/workspaces/:ws/datasets/* - Set dataset value
|
|
94
103
|
app.put('/*', async (c) => {
|
|
@@ -100,11 +109,34 @@ export function createDatasetRoutes(storage, getRepoPath, transferBackend, optio
|
|
|
100
109
|
const datasetsPrefix = `/api/repos/${encodeURIComponent(repo)}/workspaces/${encodeURIComponent(ws)}/datasets/`;
|
|
101
110
|
const pathStr = fullPath.startsWith(datasetsPrefix) ? fullPath.slice(datasetsPrefix.length) : '';
|
|
102
111
|
const treePath = urlPathToTreePath(pathStr);
|
|
103
|
-
// Body is raw BEAST2
|
|
104
|
-
|
|
105
|
-
const body = new Uint8Array(buffer);
|
|
106
|
-
return setDataset(storage, repoPath, ws, treePath, body);
|
|
112
|
+
// Body is raw BEAST2, read as it arrives.
|
|
113
|
+
return setDataset(storage, repoPath, ws, treePath, bodyChunks(c.req.raw.body));
|
|
107
114
|
});
|
|
108
115
|
return app;
|
|
109
116
|
}
|
|
117
|
+
/**
|
|
118
|
+
* A request body's chunks, as they arrive. A reader that stops early — a
|
|
119
|
+
* refused upload — cancels the body, so the connection is not left holding it.
|
|
120
|
+
*/
|
|
121
|
+
async function* bodyChunks(body) {
|
|
122
|
+
if (body === null)
|
|
123
|
+
return;
|
|
124
|
+
const reader = body.getReader();
|
|
125
|
+
let done = false;
|
|
126
|
+
try {
|
|
127
|
+
for (;;) {
|
|
128
|
+
const next = await reader.read();
|
|
129
|
+
if (next.done) {
|
|
130
|
+
done = true;
|
|
131
|
+
return;
|
|
132
|
+
}
|
|
133
|
+
yield next.value;
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
finally {
|
|
137
|
+
if (!done)
|
|
138
|
+
await reader.cancel().catch(() => { });
|
|
139
|
+
reader.releaseLock();
|
|
140
|
+
}
|
|
141
|
+
}
|
|
110
142
|
//# sourceMappingURL=datasets.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"datasets.js","sourceRoot":"","sources":["../../../src/routes/datasets.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAC5B,OAAO,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAEtD,OAAO,EACL,YAAY,EACZ,qBAAqB,EACrB,0BAA0B,EAC1B,sBAAsB,EACtB,cAAc,EACd,UAAU,EACV,cAAc,EACd,gBAAgB,EAChB,UAAU,GACX,MAAM,yBAAyB,CAAC;
|
|
1
|
+
{"version":3,"file":"datasets.js","sourceRoot":"","sources":["../../../src/routes/datasets.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAC5B,OAAO,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAEtD,OAAO,EACL,YAAY,EACZ,qBAAqB,EACrB,0BAA0B,EAC1B,sBAAsB,EACtB,cAAc,EACd,UAAU,EACV,cAAc,EACd,gBAAgB,EAChB,UAAU,GACX,MAAM,yBAAyB,CAAC;AAUjC;sDACsD;AACtD,SAAS,QAAQ,CAAC,KAAyB;IACzC,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,EAAE;QAAE,OAAO,SAAS,CAAC;IAC1D,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC;AACvB,CAAC;AAED,MAAM,UAAU,mBAAmB,CACjC,OAAuB,EACvB,WAAqC,EACrC,eAAiC,EACjC,OAA6B;IAE7B,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;IAEvB,kEAAkE;IAClE,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;QACvB,MAAM,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAE,CAAC;QAClC,MAAM,QAAQ,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;QACnC,MAAM,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAE,CAAC;QAE9B,MAAM,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,MAAM,CAAC;QAC5C,MAAM,SAAS,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,WAAW,CAAC,KAAK,MAAM,CAAC;QACtD,MAAM,MAAM,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,MAAM,CAAC;QAEhD,IAAI,SAAS,IAAI,CAAC,IAAI,EAAE,CAAC;YACvB,OAAO,CAAC,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,8BAA8B,EAAE,EAAE,GAAG,CAAC,CAAC;QAChE,CAAC;QAED,IAAI,IAAI,IAAI,SAAS,IAAI,MAAM;YAAE,OAAO,qBAAqB,CAAC,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC;QACzF,IAAI,IAAI,IAAI,SAAS;YAAY,OAAO,0BAA0B,CAAC,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC;QAC9F,IAAI,IAAI,IAAI,MAAM;YAAe,OAAO,sBAAsB,CAAC,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC;QAE1F,OAAO,YAAY,CAAC,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC;IACjD,CAAC,CAAC,CAAC;IAEH,2EAA2E;IAC3E,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;QACxB,MAAM,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAE,CAAC;QAClC,MAAM,QAAQ,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;QACnC,MAAM,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAE,CAAC;QAE9B,4DAA4D;QAC5D,MAAM,QAAQ,GAAG,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC;QAC5B,MAAM,cAAc,GAAG,cAAc,kBAAkB,CAAC,IAAI,CAAC,eAAe,kBAAkB,CAAC,EAAE,CAAC,YAAY,CAAC;QAC/G,MAAM,OAAO,GAAG,QAAQ,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACjG,MAAM,QAAQ,GAAG,iBAAiB,CAAC,OAAO,CAAC,CAAC;QAE5C,MAAM,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,MAAM,CAAC;QAC5C,MAAM,SAAS,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,WAAW,CAAC,KAAK,MAAM,CAAC;QACtD,MAAM,MAAM,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,MAAM,CAAC;QAChD,MAAM,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,MAAM,CAAC;QAC5C,MAAM,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,MAAM,CAAC;QAE5C,IAAI,SAAS,IAAI,CAAC,IAAI,EAAE,CAAC;YACvB,OAAO,CAAC,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,8BAA8B,EAAE,EAAE,GAAG,CAAC,CAAC;QAChE,CAAC;QAED,IAAI,IAAI,IAAI,SAAS,IAAI,MAAM;YAAE,OAAO,qBAAqB,CAAC,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,QAAQ,CAAC,CAAC;QAC/F,IAAI,IAAI,IAAI,SAAS;YAAY,OAAO,0BAA0B,CAAC,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,QAAQ,CAAC,CAAC;QACpG,IAAI,IAAI,IAAI,MAAM;YAAe,OAAO,sBAAsB,CAAC,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,QAAQ,CAAC,CAAC;QAChG,IAAI,IAAI;YAAyB,OAAO,YAAY,CAAC,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,QAAQ,CAAC,CAAC;QACtF,IAAI,MAAM;YAAuB,OAAO,gBAAgB,CAAC,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,QAAQ,CAAC,CAAC;QAC1F,IAAI,IAAI,EAAE,CAAC;YACT,MAAM,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;YACjC,MAAM,GAAG,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAC/B,MAAM,MAAM,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;YACrC,MAAM,MAAM,GAAG,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;YACtC,wEAAwE;YACxE,yCAAyC;YACzC,MAAM,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;YACnC,MAAM,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YAC/B,MAAM,KAAK,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YACnC,OAAO,cAAc,CAAC,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,QAAQ,EAAE;gBACrD,GAAG,CAAC,GAAG,KAAK,SAAS,IAAI,EAAE,GAAG,EAAE,CAAC;gBACjC,GAAG,CAAC,MAAM,KAAK,SAAS,IAAI,EAAE,MAAM,EAAE,CAAC;gBACvC,GAAG,CAAC,MAAM,KAAK,SAAS,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC;gBAC5D,GAAG,CAAC,IAAI,KAAK,SAAS,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC;gBACtD,GAAG,CAAC,EAAE,KAAK,SAAS,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,EAAE,EAAE,CAAC;gBAChD,GAAG,CAAC,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;gBACrD,GAAG,CAAC,IAAI,KAAK,SAAS,IAAI,IAAI,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;aACnD,CAAC,CAAC;QACL,CAAC;QACD,IAAI,IAAI,EAAE,CAAC;YACT,MAAM,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;YACjC,MAAM,MAAM,GAAG;gBACb,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,KAAK,SAAS,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAE,EAAE,CAAC;gBAClG,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,KAAK,SAAS,IAAI,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,CAAE,EAAE,CAAC;gBAC/F,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,KAAK,SAAS,IAAI,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC,CAAE,EAAE,CAAC;gBACrG,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,SAAS,IAAI,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAE,EAAE,CAAC;gBAC1G,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,MAAM,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;gBACrD,GAAG,CAAC,IAAI,KAAK,SAAS,IAAI,IAAI,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;aACnD,CAAC;YACF,OAAO,cAAc,CAAC,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE;gBAC7D,GAAG,CAAC,OAAO,EAAE,cAAc,KAAK,SAAS,IAAI,EAAE,UAAU,EAAE,OAAO,CAAC,cAAc,EAAE,CAAC;aACrF,CAAC,CAAC;QACL,CAAC;QAED,uEAAuE;QACvE,sEAAsE;QACtE,OAAO,UAAU,CAAC,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,GAAG,EAAE,eAAe,EAAE,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,UAAU,CAAC,KAAK,MAAM,CAAC,CAAC;IAC3H,CAAC,CAAC,CAAC;IAEH,qEAAqE;IACrE,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;QACxB,MAAM,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAE,CAAC;QAClC,MAAM,QAAQ,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;QACnC,MAAM,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAE,CAAC;QAE9B,4DAA4D;QAC5D,MAAM,QAAQ,GAAG,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC;QAC5B,MAAM,cAAc,GAAG,cAAc,kBAAkB,CAAC,IAAI,CAAC,eAAe,kBAAkB,CAAC,EAAE,CAAC,YAAY,CAAC;QAC/G,MAAM,OAAO,GAAG,QAAQ,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACjG,MAAM,QAAQ,GAAG,iBAAiB,CAAC,OAAO,CAAC,CAAC;QAE5C,0CAA0C;QAC1C,OAAO,UAAU,CAAC,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,QAAQ,EAAE,UAAU,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;IACjF,CAAC,CAAC,CAAC;IAEH,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;GAGG;AACH,KAAK,SAAS,CAAC,CAAC,UAAU,CAAC,IAAuC;IAChE,IAAI,IAAI,KAAK,IAAI;QAAE,OAAO;IAC1B,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;IAChC,IAAI,IAAI,GAAG,KAAK,CAAC;IACjB,IAAI,CAAC;QACH,SAAS,CAAC;YACR,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;YACjC,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;gBACd,IAAI,GAAG,IAAI,CAAC;gBACZ,OAAO;YACT,CAAC;YACD,MAAM,IAAI,CAAC,KAAK,CAAC;QACnB,CAAC;IACH,CAAC;YAAS,CAAC;QACT,IAAI,CAAC,IAAI;YAAE,MAAM,MAAM,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAAyC,CAAC,CAAC,CAAC;QACxF,MAAM,CAAC,WAAW,EAAE,CAAC;IACvB,CAAC;AACH,CAAC"}
|
|
@@ -3,6 +3,41 @@
|
|
|
3
3
|
* Licensed under BSL 1.1. See LICENSE for details.
|
|
4
4
|
*/
|
|
5
5
|
import { Hono } from 'hono';
|
|
6
|
-
import {
|
|
7
|
-
|
|
6
|
+
import type { Budget, DataflowOrchestrator, ExecutionStateStore, StorageBackend } from '@elaraai/e3-core';
|
|
7
|
+
import type { GetRunner } from './functions.js';
|
|
8
|
+
/**
|
|
9
|
+
* The seams the dataflow routes go through, which the host that mounts them
|
|
10
|
+
* gives: what runs a repository's dataflows, and where their runs are kept.
|
|
11
|
+
* A local server's are a LocalOrchestrator and a FileStateStore for each
|
|
12
|
+
* repository; a cloud's run its dataflows on its own compute, and keep their
|
|
13
|
+
* state in its own store.
|
|
14
|
+
*/
|
|
15
|
+
export interface DataflowSeams {
|
|
16
|
+
/** The runner a repository's tasks and units run on, which the status also
|
|
17
|
+
* asks whether an execution recorded running can still finish */
|
|
18
|
+
getRunner: GetRunner;
|
|
19
|
+
/** The orchestrator that runs a repository's dataflows, and cancels one */
|
|
20
|
+
getOrchestrator: (repoPath: string) => DataflowOrchestrator;
|
|
21
|
+
/** The store a repository's runs keep their state in, which a poll and a
|
|
22
|
+
* cancel read: the one the orchestrator writes */
|
|
23
|
+
getStateStore: (repoPath: string) => ExecutionStateStore;
|
|
24
|
+
/** The tasks and units the loop keeps in flight; the orchestrator's own
|
|
25
|
+
* default when absent */
|
|
26
|
+
width?: number;
|
|
27
|
+
/** The budget the runner holds, which the poll and the budget route serve;
|
|
28
|
+
* absent for a host whose runners hold none */
|
|
29
|
+
budget?: Budget;
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* The routes of a workspace's dataflow: start, poll, cancel, its graph, its
|
|
33
|
+
* tasks' logs, and the budget a run gets.
|
|
34
|
+
*
|
|
35
|
+
* @param storage - Storage backend
|
|
36
|
+
* @param getRepoPath - A repository's path from its name
|
|
37
|
+
* @param seams - What runs a repository's dataflows, and where their runs are
|
|
38
|
+
* kept: a poll or a cancel reads the run's state from the store, whichever
|
|
39
|
+
* instance answers it
|
|
40
|
+
* @returns The routes
|
|
41
|
+
*/
|
|
42
|
+
export declare function createExecutionRoutes(storage: StorageBackend, getRepoPath: (repo: string) => string, seams: DataflowSeams): Hono<import("hono/types").BlankEnv, import("hono/types").BlankSchema, "/">;
|
|
8
43
|
//# sourceMappingURL=executions.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"executions.d.ts","sourceRoot":"","sources":["../../../src/routes/executions.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAC5B,OAAO,
|
|
1
|
+
{"version":3,"file":"executions.d.ts","sourceRoot":"","sources":["../../../src/routes/executions.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAC5B,OAAO,KAAK,EAAE,MAAM,EAAE,oBAAoB,EAAE,mBAAmB,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAY1G,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAEhD;;;;;;GAMG;AACH,MAAM,WAAW,aAAa;IAC5B;sEACkE;IAClE,SAAS,EAAE,SAAS,CAAC;IACrB,2EAA2E;IAC3E,eAAe,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,oBAAoB,CAAC;IAC5D;uDACmD;IACnD,aAAa,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,mBAAmB,CAAC;IACzD;8BAC0B;IAC1B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;oDACgD;IAChD,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,qBAAqB,CACnC,OAAO,EAAE,cAAc,EACvB,WAAW,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,MAAM,EACrC,KAAK,EAAE,aAAa,8EA+ErB"}
|
|
@@ -3,11 +3,21 @@
|
|
|
3
3
|
* Licensed under BSL 1.1. See LICENSE for details.
|
|
4
4
|
*/
|
|
5
5
|
import { Hono } from 'hono';
|
|
6
|
-
import {
|
|
7
|
-
import { startDataflow, getDataflowStatus, getDataflowGraph, getTaskLogs, getDataflowExecution, cancelDataflow, } from '../handlers/dataflow.js';
|
|
6
|
+
import { startDataflow, getDataflowStatus, getDataflowGraph, getTaskLogs, getDataflowExecution, getDataflowBudget, cancelDataflow, } from '../handlers/dataflow.js';
|
|
8
7
|
import { decodeBody } from '../beast2.js';
|
|
9
8
|
import { DataflowRequestType } from '../types.js';
|
|
10
|
-
|
|
9
|
+
/**
|
|
10
|
+
* The routes of a workspace's dataflow: start, poll, cancel, its graph, its
|
|
11
|
+
* tasks' logs, and the budget a run gets.
|
|
12
|
+
*
|
|
13
|
+
* @param storage - Storage backend
|
|
14
|
+
* @param getRepoPath - A repository's path from its name
|
|
15
|
+
* @param seams - What runs a repository's dataflows, and where their runs are
|
|
16
|
+
* kept: a poll or a cancel reads the run's state from the store, whichever
|
|
17
|
+
* instance answers it
|
|
18
|
+
* @returns The routes
|
|
19
|
+
*/
|
|
20
|
+
export function createExecutionRoutes(storage, getRepoPath, seams) {
|
|
11
21
|
const app = new Hono();
|
|
12
22
|
// POST /api/repos/:repo/workspaces/:ws/dataflow - Start dataflow execution (non-blocking)
|
|
13
23
|
app.post('/', async (c) => {
|
|
@@ -15,13 +25,10 @@ export function createExecutionRoutes(storage, getRepoPath) {
|
|
|
15
25
|
const repoPath = getRepoPath(repo);
|
|
16
26
|
const ws = c.req.param('ws');
|
|
17
27
|
const body = await decodeBody(c, DataflowRequestType);
|
|
18
|
-
// The request's `concurrency` is the run's jobs budget: the runner
|
|
19
|
-
// processes this server keeps in flight for it, across tasks and the
|
|
20
|
-
// units of partitioned tasks. Absent, the CPUs available to the server.
|
|
21
|
-
const jobs = body.concurrency.type === 'some' ? Number(body.concurrency.value) : defaultJobs();
|
|
22
28
|
const filter = body.filter.type === 'some' ? body.filter.value : undefined;
|
|
23
|
-
return startDataflow(storage, repoPath, ws, {
|
|
24
|
-
|
|
29
|
+
return startDataflow(storage, seams.getOrchestrator(repoPath), repoPath, ws, {
|
|
30
|
+
runner: seams.getRunner(repoPath),
|
|
31
|
+
...(seams.width !== undefined && { width: seams.width }),
|
|
25
32
|
force: body.force,
|
|
26
33
|
filter,
|
|
27
34
|
verbose: c.req.query('verbose') === '1',
|
|
@@ -32,7 +39,7 @@ export function createExecutionRoutes(storage, getRepoPath) {
|
|
|
32
39
|
const repo = c.req.param('repo');
|
|
33
40
|
const repoPath = getRepoPath(repo);
|
|
34
41
|
const ws = c.req.param('ws');
|
|
35
|
-
return getDataflowStatus(storage, repoPath, ws);
|
|
42
|
+
return getDataflowStatus(storage, seams.getRunner(repoPath), repoPath, ws);
|
|
36
43
|
});
|
|
37
44
|
// GET /api/repos/:repo/workspaces/:ws/dataflow/graph - Get dependency graph
|
|
38
45
|
app.get('/graph', async (c) => {
|
|
@@ -61,14 +68,16 @@ export function createExecutionRoutes(storage, getRepoPath) {
|
|
|
61
68
|
// Get query params for pagination
|
|
62
69
|
const offset = c.req.query('offset') ? parseInt(c.req.query('offset'), 10) : undefined;
|
|
63
70
|
const limit = c.req.query('limit') ? parseInt(c.req.query('limit'), 10) : undefined;
|
|
64
|
-
return getDataflowExecution(repoPath, ws, { offset, limit });
|
|
71
|
+
return getDataflowExecution(seams.getStateStore(repoPath), seams.getOrchestrator(repoPath), repoPath, ws, { offset, limit }, seams.budget);
|
|
65
72
|
});
|
|
73
|
+
// GET /api/repos/:repo/workspaces/:ws/dataflow/budget - The budget a run gets
|
|
74
|
+
app.get('/budget', () => getDataflowBudget(seams.budget));
|
|
66
75
|
// POST /api/repos/:repo/workspaces/:ws/dataflow/cancel - Cancel running execution
|
|
67
76
|
app.post('/cancel', async (c) => {
|
|
68
77
|
const repo = c.req.param('repo');
|
|
69
78
|
const repoPath = getRepoPath(repo);
|
|
70
79
|
const ws = c.req.param('ws');
|
|
71
|
-
return cancelDataflow(repoPath, ws);
|
|
80
|
+
return cancelDataflow(seams.getStateStore(repoPath), seams.getOrchestrator(repoPath), repoPath, ws);
|
|
72
81
|
});
|
|
73
82
|
return app;
|
|
74
83
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"executions.js","sourceRoot":"","sources":["../../../src/routes/executions.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"executions.js","sourceRoot":"","sources":["../../../src/routes/executions.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAE5B,OAAO,EACL,aAAa,EACb,iBAAiB,EACjB,gBAAgB,EAChB,WAAW,EACX,oBAAoB,EACpB,iBAAiB,EACjB,cAAc,GACf,MAAM,yBAAyB,CAAC;AACjC,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAC1C,OAAO,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AA2BlD;;;;;;;;;;GAUG;AACH,MAAM,UAAU,qBAAqB,CACnC,OAAuB,EACvB,WAAqC,EACrC,KAAoB;IAEpB,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;IAEvB,0FAA0F;IAC1F,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;QACxB,MAAM,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAE,CAAC;QAClC,MAAM,QAAQ,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;QACnC,MAAM,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAE,CAAC;QAE9B,MAAM,IAAI,GAAG,MAAM,UAAU,CAAC,CAAC,EAAE,mBAAmB,CAAC,CAAC;QACtD,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;QAE3E,OAAO,aAAa,CAAC,OAAO,EAAE,KAAK,CAAC,eAAe,CAAC,QAAQ,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE;YAC3E,MAAM,EAAE,KAAK,CAAC,SAAS,CAAC,QAAQ,CAAC;YACjC,GAAG,CAAC,KAAK,CAAC,KAAK,KAAK,SAAS,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC;YACxD,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,MAAM;YACN,OAAO,EAAE,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC,KAAK,GAAG;SACxC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,oFAAoF;IACpF,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;QACvB,MAAM,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAE,CAAC;QAClC,MAAM,QAAQ,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;QACnC,MAAM,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAE,CAAC;QAC9B,OAAO,iBAAiB,CAAC,OAAO,EAAE,KAAK,CAAC,SAAS,CAAC,QAAQ,CAAC,EAAE,QAAQ,EAAE,EAAE,CAAC,CAAC;IAC7E,CAAC,CAAC,CAAC;IAEH,4EAA4E;IAC5E,GAAG,CAAC,GAAG,CAAC,QAAQ,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;QAC5B,MAAM,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAE,CAAC;QAClC,MAAM,QAAQ,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;QACnC,MAAM,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAE,CAAC;QAC9B,OAAO,gBAAgB,CAAC,OAAO,EAAE,QAAQ,EAAE,EAAE,CAAC,CAAC;IACjD,CAAC,CAAC,CAAC;IAEH,0EAA0E;IAC1E,GAAG,CAAC,GAAG,CAAC,aAAa,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;QACjC,MAAM,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAE,CAAC;QAClC,MAAM,QAAQ,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;QACnC,MAAM,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAE,CAAC;QAC9B,MAAM,QAAQ,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAE,CAAC;QAEtC,mBAAmB;QACnB,MAAM,MAAM,GAAI,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAyB,IAAI,QAAQ,CAAC;QAC1E,MAAM,MAAM,GAAG,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,GAAG,EAAE,EAAE,CAAC,CAAC;QAC1D,MAAM,KAAK,GAAG,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,OAAO,EAAE,EAAE,CAAC,CAAC;QAE5D,OAAO,WAAW,CAAC,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC;IAC7E,CAAC,CAAC,CAAC;IAEH,6FAA6F;IAC7F,GAAG,CAAC,GAAG,CAAC,YAAY,EAAE,CAAC,CAAC,EAAE,EAAE;QAC1B,MAAM,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAE,CAAC;QAClC,MAAM,QAAQ,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;QACnC,MAAM,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAE,CAAC;QAE9B,kCAAkC;QAClC,MAAM,MAAM,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAE,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QACxF,MAAM,KAAK,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAE,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QAErF,OAAO,oBAAoB,CAAC,KAAK,CAAC,aAAa,CAAC,QAAQ,CAAC,EAAE,KAAK,CAAC,eAAe,CAAC,QAAQ,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IAC7I,CAAC,CAAC,CAAC;IAEH,8EAA8E;IAC9E,GAAG,CAAC,GAAG,CAAC,SAAS,EAAE,GAAG,EAAE,CAAC,iBAAiB,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;IAE1D,kFAAkF;IAClF,GAAG,CAAC,IAAI,CAAC,SAAS,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;QAC9B,MAAM,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAE,CAAC;QAClC,MAAM,QAAQ,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;QACnC,MAAM,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAE,CAAC;QAE9B,OAAO,cAAc,CAAC,KAAK,CAAC,aAAa,CAAC,QAAQ,CAAC,EAAE,KAAK,CAAC,eAAe,CAAC,QAAQ,CAAC,EAAE,QAAQ,EAAE,EAAE,CAAC,CAAC;IACtG,CAAC,CAAC,CAAC;IAEH,OAAO,GAAG,CAAC;AACb,CAAC"}
|
|
@@ -2,12 +2,13 @@
|
|
|
2
2
|
* Copyright (c) 2025 Elara AI Pty Ltd
|
|
3
3
|
* Licensed under BSL 1.1. See LICENSE for details.
|
|
4
4
|
*/
|
|
5
|
+
export { createRepositoriesRoutes, createSingleRepositoryRoutes } from './repositories.js';
|
|
5
6
|
export { createRepositoryRoutes } from './repository.js';
|
|
6
7
|
export { createPackageRoutes } from './packages.js';
|
|
7
8
|
export { createWorkspaceRoutes } from './workspaces.js';
|
|
8
9
|
export { createDatasetRoutes } from './datasets.js';
|
|
9
10
|
export { createTaskRoutes } from './tasks.js';
|
|
10
|
-
export { createExecutionRoutes } from './executions.js';
|
|
11
|
+
export { createExecutionRoutes, type DataflowSeams } from './executions.js';
|
|
11
12
|
export { createObjectRoutes } from './objects.js';
|
|
12
13
|
export { createTransferRoutes, type TransferRouteOptions } from './transfer.js';
|
|
13
14
|
export { createPackageTransferRoutes } from './package-transfer.js';
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/routes/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,sBAAsB,EAAE,MAAM,iBAAiB,CAAC;AACzD,OAAO,EAAE,mBAAmB,EAAE,MAAM,eAAe,CAAC;AACpD,OAAO,EAAE,qBAAqB,EAAE,MAAM,iBAAiB,CAAC;AACxD,OAAO,EAAE,mBAAmB,EAAE,MAAM,eAAe,CAAC;AACpD,OAAO,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAC9C,OAAO,EAAE,qBAAqB,EAAE,MAAM,iBAAiB,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/routes/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,wBAAwB,EAAE,4BAA4B,EAAE,MAAM,mBAAmB,CAAC;AAC3F,OAAO,EAAE,sBAAsB,EAAE,MAAM,iBAAiB,CAAC;AACzD,OAAO,EAAE,mBAAmB,EAAE,MAAM,eAAe,CAAC;AACpD,OAAO,EAAE,qBAAqB,EAAE,MAAM,iBAAiB,CAAC;AACxD,OAAO,EAAE,mBAAmB,EAAE,MAAM,eAAe,CAAC;AACpD,OAAO,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAC9C,OAAO,EAAE,qBAAqB,EAAE,KAAK,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAC5E,OAAO,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAC;AAClD,OAAO,EAAE,oBAAoB,EAAE,KAAK,oBAAoB,EAAE,MAAM,eAAe,CAAC;AAChF,OAAO,EAAE,2BAA2B,EAAE,MAAM,uBAAuB,CAAC;AACpE,OAAO,EAAE,mBAAmB,EAAE,MAAM,WAAW,CAAC;AAChD,OAAO,EAAE,2BAA2B,EAAE,6BAA6B,EAAE,mBAAmB,EAAE,KAAK,SAAS,EAAE,MAAM,gBAAgB,CAAC;AACjI,OAAO,EAAE,2BAA2B,EAAE,MAAM,cAAc,CAAC"}
|
package/dist/src/routes/index.js
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
* Copyright (c) 2025 Elara AI Pty Ltd
|
|
3
3
|
* Licensed under BSL 1.1. See LICENSE for details.
|
|
4
4
|
*/
|
|
5
|
+
export { createRepositoriesRoutes, createSingleRepositoryRoutes } from './repositories.js';
|
|
5
6
|
export { createRepositoryRoutes } from './repository.js';
|
|
6
7
|
export { createPackageRoutes } from './packages.js';
|
|
7
8
|
export { createWorkspaceRoutes } from './workspaces.js';
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/routes/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,sBAAsB,EAAE,MAAM,iBAAiB,CAAC;AACzD,OAAO,EAAE,mBAAmB,EAAE,MAAM,eAAe,CAAC;AACpD,OAAO,EAAE,qBAAqB,EAAE,MAAM,iBAAiB,CAAC;AACxD,OAAO,EAAE,mBAAmB,EAAE,MAAM,eAAe,CAAC;AACpD,OAAO,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAC9C,OAAO,EAAE,qBAAqB,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/routes/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,wBAAwB,EAAE,4BAA4B,EAAE,MAAM,mBAAmB,CAAC;AAC3F,OAAO,EAAE,sBAAsB,EAAE,MAAM,iBAAiB,CAAC;AACzD,OAAO,EAAE,mBAAmB,EAAE,MAAM,eAAe,CAAC;AACpD,OAAO,EAAE,qBAAqB,EAAE,MAAM,iBAAiB,CAAC;AACxD,OAAO,EAAE,mBAAmB,EAAE,MAAM,eAAe,CAAC;AACpD,OAAO,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAC9C,OAAO,EAAE,qBAAqB,EAAsB,MAAM,iBAAiB,CAAC;AAC5E,OAAO,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAC;AAClD,OAAO,EAAE,oBAAoB,EAA6B,MAAM,eAAe,CAAC;AAChF,OAAO,EAAE,2BAA2B,EAAE,MAAM,uBAAuB,CAAC;AACpE,OAAO,EAAE,mBAAmB,EAAE,MAAM,WAAW,CAAC;AAChD,OAAO,EAAE,2BAA2B,EAAE,6BAA6B,EAAE,mBAAmB,EAAkB,MAAM,gBAAgB,CAAC;AACjI,OAAO,EAAE,2BAA2B,EAAE,MAAM,cAAc,CAAC"}
|
|
@@ -3,6 +3,20 @@
|
|
|
3
3
|
* Licensed under BSL 1.1. See LICENSE for details.
|
|
4
4
|
*/
|
|
5
5
|
import { Hono } from 'hono';
|
|
6
|
-
import { type StorageBackend } from '@elaraai/e3-core';
|
|
7
|
-
|
|
6
|
+
import { type StorageBackend, type TransferBackend } from '@elaraai/e3-core';
|
|
7
|
+
/**
|
|
8
|
+
* Creates the routes that read an object by its hash.
|
|
9
|
+
*
|
|
10
|
+
* @remarks
|
|
11
|
+
* A client downloading a collection by its segments reads each segment here.
|
|
12
|
+
* With a transfer backend, an object over {@link DOWNLOAD_REDIRECT_BYTES} is
|
|
13
|
+
* answered as a dataset download is: JSON `{ url }`, which the client fetches
|
|
14
|
+
* directly, so a host whose responses are capped never carries the bytes.
|
|
15
|
+
*
|
|
16
|
+
* @param storage - The storage backend the objects are read from
|
|
17
|
+
* @param getRepoPath - Maps a repository name to its path in the backend
|
|
18
|
+
* @param transferBackend - Serves large objects by URL; without it every object is answered inline
|
|
19
|
+
* @returns The Hono app, mounted at `/api/repos/:repo/objects`
|
|
20
|
+
*/
|
|
21
|
+
export declare function createObjectRoutes(storage: StorageBackend, getRepoPath: (repo: string) => string, transferBackend?: TransferBackend): Hono<import("hono/types").BlankEnv, import("hono/types").BlankSchema, "/">;
|
|
8
22
|
//# sourceMappingURL=objects.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"objects.d.ts","sourceRoot":"","sources":["../../../src/routes/objects.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAC5B,OAAO,
|
|
1
|
+
{"version":3,"file":"objects.d.ts","sourceRoot":"","sources":["../../../src/routes/objects.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAC5B,OAAO,EAAgB,KAAK,cAAc,EAAE,KAAK,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAK3F;;;;;;;;;;;;;GAaG;AACH,wBAAgB,kBAAkB,CAChC,OAAO,EAAE,cAAc,EACvB,WAAW,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,MAAM,EACrC,eAAe,CAAC,EAAE,eAAe,8EA8ClC"}
|