@elaraai/e3-api-server 1.0.79 → 1.0.81
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +100 -20
- package/dist/src/beast2.js +1 -1
- package/dist/src/beast2.js.map +1 -1
- package/dist/src/cli.js +13 -0
- package/dist/src/cli.js.map +1 -1
- package/dist/src/errors.d.ts.map +1 -1
- package/dist/src/errors.js +8 -1
- package/dist/src/errors.js.map +1 -1
- package/dist/src/handlers/dataflow.d.ts +58 -11
- package/dist/src/handlers/dataflow.d.ts.map +1 -1
- package/dist/src/handlers/dataflow.js +120 -48
- package/dist/src/handlers/dataflow.js.map +1 -1
- package/dist/src/handlers/dataflow.spec.d.ts +6 -0
- package/dist/src/handlers/dataflow.spec.d.ts.map +1 -0
- package/dist/src/handlers/dataflow.spec.js +106 -0
- package/dist/src/handlers/dataflow.spec.js.map +1 -0
- package/dist/src/handlers/datasets.d.ts +60 -34
- package/dist/src/handlers/datasets.d.ts.map +1 -1
- package/dist/src/handlers/datasets.js +452 -306
- package/dist/src/handlers/datasets.js.map +1 -1
- package/dist/src/handlers/datasets.spec.js +338 -119
- package/dist/src/handlers/datasets.spec.js.map +1 -1
- package/dist/src/handlers/functions.d.ts.map +1 -1
- package/dist/src/handlers/functions.js +10 -8
- package/dist/src/handlers/functions.js.map +1 -1
- package/dist/src/handlers/functions.spec.js +2 -2
- package/dist/src/handlers/functions.spec.js.map +1 -1
- package/dist/src/handlers/index.d.ts +2 -2
- package/dist/src/handlers/index.d.ts.map +1 -1
- package/dist/src/handlers/index.js +2 -2
- package/dist/src/handlers/index.js.map +1 -1
- package/dist/src/handlers/records.d.ts.map +1 -1
- package/dist/src/handlers/records.js +15 -12
- package/dist/src/handlers/records.js.map +1 -1
- package/dist/src/handlers/records.spec.js +9 -5
- package/dist/src/handlers/records.spec.js.map +1 -1
- package/dist/src/handlers/repository.d.ts +59 -10
- package/dist/src/handlers/repository.d.ts.map +1 -1
- package/dist/src/handlers/repository.js +147 -40
- package/dist/src/handlers/repository.js.map +1 -1
- package/dist/src/handlers/tasks.d.ts +2 -1
- package/dist/src/handlers/tasks.d.ts.map +1 -1
- package/dist/src/handlers/tasks.js +31 -8
- package/dist/src/handlers/tasks.js.map +1 -1
- package/dist/src/handlers/tasks.spec.d.ts +6 -0
- package/dist/src/handlers/tasks.spec.d.ts.map +1 -0
- package/dist/src/handlers/tasks.spec.js +64 -0
- package/dist/src/handlers/tasks.spec.js.map +1 -0
- package/dist/src/handlers/workspaces.d.ts +63 -6
- package/dist/src/handlers/workspaces.d.ts.map +1 -1
- package/dist/src/handlers/workspaces.js +104 -42
- package/dist/src/handlers/workspaces.js.map +1 -1
- package/dist/src/handlers/workspaces.spec.d.ts +6 -0
- package/dist/src/handlers/workspaces.spec.d.ts.map +1 -0
- package/dist/src/handlers/workspaces.spec.js +214 -0
- package/dist/src/handlers/workspaces.spec.js.map +1 -0
- package/dist/src/index.d.ts +3 -0
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +7 -0
- package/dist/src/index.js.map +1 -1
- package/dist/src/local-dataflow.d.ts +14 -0
- package/dist/src/local-dataflow.d.ts.map +1 -0
- package/dist/src/local-dataflow.js +41 -0
- package/dist/src/local-dataflow.js.map +1 -0
- package/dist/src/middleware/repository.d.ts +39 -0
- package/dist/src/middleware/repository.d.ts.map +1 -0
- package/dist/src/middleware/repository.js +77 -0
- package/dist/src/middleware/repository.js.map +1 -0
- package/dist/src/routes/data.d.ts +2 -2
- package/dist/src/routes/data.d.ts.map +1 -1
- package/dist/src/routes/data.js +22 -45
- package/dist/src/routes/data.js.map +1 -1
- package/dist/src/routes/datasets.d.ts +0 -3
- package/dist/src/routes/datasets.d.ts.map +1 -1
- package/dist/src/routes/datasets.js +40 -8
- package/dist/src/routes/datasets.js.map +1 -1
- package/dist/src/routes/executions.d.ts +37 -2
- package/dist/src/routes/executions.d.ts.map +1 -1
- package/dist/src/routes/executions.js +21 -12
- package/dist/src/routes/executions.js.map +1 -1
- package/dist/src/routes/index.d.ts +2 -1
- package/dist/src/routes/index.d.ts.map +1 -1
- package/dist/src/routes/index.js +1 -0
- package/dist/src/routes/index.js.map +1 -1
- package/dist/src/routes/objects.d.ts +16 -2
- package/dist/src/routes/objects.d.ts.map +1 -1
- package/dist/src/routes/objects.js +34 -3
- package/dist/src/routes/objects.js.map +1 -1
- package/dist/src/routes/repositories.d.ts +26 -0
- package/dist/src/routes/repositories.d.ts.map +1 -0
- package/dist/src/routes/repositories.js +58 -0
- package/dist/src/routes/repositories.js.map +1 -0
- package/dist/src/routes/repositories.spec.d.ts +6 -0
- package/dist/src/routes/repositories.spec.d.ts.map +1 -0
- package/dist/src/routes/repositories.spec.js +77 -0
- package/dist/src/routes/repositories.spec.js.map +1 -0
- package/dist/src/routes/repository.d.ts +12 -2
- package/dist/src/routes/repository.d.ts.map +1 -1
- package/dist/src/routes/repository.js +24 -9
- package/dist/src/routes/repository.js.map +1 -1
- package/dist/src/routes/transfer.d.ts +22 -10
- package/dist/src/routes/transfer.d.ts.map +1 -1
- package/dist/src/routes/transfer.js +104 -118
- package/dist/src/routes/transfer.js.map +1 -1
- package/dist/src/routes/workspaces.d.ts +14 -1
- package/dist/src/routes/workspaces.d.ts.map +1 -1
- package/dist/src/routes/workspaces.js +64 -46
- package/dist/src/routes/workspaces.js.map +1 -1
- package/dist/src/server.d.ts +18 -6
- package/dist/src/server.d.ts.map +1 -1
- package/dist/src/server.js +57 -183
- package/dist/src/server.js.map +1 -1
- package/dist/src/server.spec.d.ts +6 -0
- package/dist/src/server.spec.d.ts.map +1 -0
- package/dist/src/server.spec.js +59 -0
- package/dist/src/server.spec.js.map +1 -0
- package/dist/src/types.d.ts +306 -17
- package/dist/src/types.d.ts.map +1 -1
- package/dist/src/types.js +7 -3
- package/dist/src/types.js.map +1 -1
- package/package.json +5 -5
- package/dist/src/async-operation-state.d.ts +0 -37
- package/dist/src/async-operation-state.d.ts.map +0 -1
- package/dist/src/async-operation-state.js +0 -118
- package/dist/src/async-operation-state.js.map +0 -1
- package/dist/src/orchestrator-manager.d.ts +0 -45
- package/dist/src/orchestrator-manager.d.ts.map +0 -1
- package/dist/src/orchestrator-manager.js +0 -150
- package/dist/src/orchestrator-manager.js.map +0 -1
package/dist/src/index.js
CHANGED
|
@@ -10,10 +10,17 @@
|
|
|
10
10
|
*/
|
|
11
11
|
export { createServer } from './server.js';
|
|
12
12
|
export { ApiTypes } from './types.js';
|
|
13
|
+
// How `ServerConfig.budget` settings resolve, for an embedder that checks them
|
|
14
|
+
// before it starts a server
|
|
15
|
+
export { resolveBudget } from '@elaraai/e3-core';
|
|
13
16
|
// Export handlers for Lambda reuse
|
|
14
17
|
export * from './handlers/index.js';
|
|
18
|
+
// The route factories, which another host mounts with seams of its own
|
|
19
|
+
export * from './routes/index.js';
|
|
15
20
|
// Export auth middleware
|
|
16
21
|
export { createAuthMiddleware } from './middleware/auth.js';
|
|
22
|
+
// The gate every request to one repository passes
|
|
23
|
+
export { createRepositoryGate, createSingleRepositoryGate } from './middleware/repository.js';
|
|
17
24
|
// Export BEAST2 helpers for custom integrations
|
|
18
25
|
export { sendSuccess, sendError, sendSuccessWithStatus, decodeBeast2, decodeBody } from './beast2.js';
|
|
19
26
|
//# sourceMappingURL=index.js.map
|
package/dist/src/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;;GAKG;AAEH,OAAO,EAAE,YAAY,EAAkC,MAAM,aAAa,CAAC;AAC3E,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAEtC,mCAAmC;AACnC,cAAc,qBAAqB,CAAC;AAEpC,yBAAyB;AACzB,OAAO,EAAE,oBAAoB,EAAmB,MAAM,sBAAsB,CAAC;AAE7E,gDAAgD;AAChD,OAAO,EAAE,WAAW,EAAE,SAAS,EAAE,qBAAqB,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;;GAKG;AAEH,OAAO,EAAE,YAAY,EAAkC,MAAM,aAAa,CAAC;AAC3E,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAEtC,+EAA+E;AAC/E,4BAA4B;AAC5B,OAAO,EAAE,aAAa,EAAuB,MAAM,kBAAkB,CAAC;AAEtE,mCAAmC;AACnC,cAAc,qBAAqB,CAAC;AAEpC,uEAAuE;AACvE,cAAc,mBAAmB,CAAC;AAElC,yBAAyB;AACzB,OAAO,EAAE,oBAAoB,EAAmB,MAAM,sBAAsB,CAAC;AAE7E,kDAAkD;AAClD,OAAO,EAAE,oBAAoB,EAAE,0BAA0B,EAAE,MAAM,4BAA4B,CAAC;AAE9F,gDAAgD;AAChD,OAAO,EAAE,WAAW,EAAE,SAAS,EAAE,qBAAqB,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC"}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Copyright (c) 2025 Elara AI Pty Ltd
|
|
3
|
+
* Licensed under BSL 1.1. See LICENSE for details.
|
|
4
|
+
*/
|
|
5
|
+
import type { DataflowSeams } from './routes/executions.js';
|
|
6
|
+
/**
|
|
7
|
+
* The orchestrator and state store of each repository a local server serves,
|
|
8
|
+
* each made as its repository is first asked for and kept for the server's
|
|
9
|
+
* life, so a poll or a cancel reaches the run its start began.
|
|
10
|
+
*
|
|
11
|
+
* @returns The seams' getters, for the dataflow routes
|
|
12
|
+
*/
|
|
13
|
+
export declare function localDataflow(): Pick<DataflowSeams, 'getOrchestrator' | 'getStateStore'>;
|
|
14
|
+
//# sourceMappingURL=local-dataflow.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"local-dataflow.d.ts","sourceRoot":"","sources":["../../src/local-dataflow.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAgBH,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AAE5D;;;;;;GAMG;AACH,wBAAgB,aAAa,IAAI,IAAI,CAAC,aAAa,EAAE,iBAAiB,GAAG,eAAe,CAAC,CAuBxF"}
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Copyright (c) 2025 Elara AI Pty Ltd
|
|
3
|
+
* Licensed under BSL 1.1. See LICENSE for details.
|
|
4
|
+
*/
|
|
5
|
+
/**
|
|
6
|
+
* The local server's dataflow wiring: a LocalOrchestrator for each repository
|
|
7
|
+
* it serves, over a FileStateStore in the repository's workspaces directory.
|
|
8
|
+
* The dataflow routes run, poll and cancel a repository's dataflows through
|
|
9
|
+
* them, as a cloud's routes do through its own.
|
|
10
|
+
*/
|
|
11
|
+
import { join } from 'node:path';
|
|
12
|
+
import { LocalOrchestrator, FileStateStore, } from '@elaraai/e3-core';
|
|
13
|
+
/**
|
|
14
|
+
* The orchestrator and state store of each repository a local server serves,
|
|
15
|
+
* each made as its repository is first asked for and kept for the server's
|
|
16
|
+
* life, so a poll or a cancel reaches the run its start began.
|
|
17
|
+
*
|
|
18
|
+
* @returns The seams' getters, for the dataflow routes
|
|
19
|
+
*/
|
|
20
|
+
export function localDataflow() {
|
|
21
|
+
const stateStores = new Map();
|
|
22
|
+
const orchestrators = new Map();
|
|
23
|
+
const getStateStore = (repoPath) => {
|
|
24
|
+
let store = stateStores.get(repoPath);
|
|
25
|
+
if (store === undefined) {
|
|
26
|
+
store = new FileStateStore(join(repoPath, 'workspaces'));
|
|
27
|
+
stateStores.set(repoPath, store);
|
|
28
|
+
}
|
|
29
|
+
return store;
|
|
30
|
+
};
|
|
31
|
+
const getOrchestrator = (repoPath) => {
|
|
32
|
+
let orchestrator = orchestrators.get(repoPath);
|
|
33
|
+
if (orchestrator === undefined) {
|
|
34
|
+
orchestrator = new LocalOrchestrator(getStateStore(repoPath));
|
|
35
|
+
orchestrators.set(repoPath, orchestrator);
|
|
36
|
+
}
|
|
37
|
+
return orchestrator;
|
|
38
|
+
};
|
|
39
|
+
return { getOrchestrator, getStateStore };
|
|
40
|
+
}
|
|
41
|
+
//# sourceMappingURL=local-dataflow.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"local-dataflow.js","sourceRoot":"","sources":["../../src/local-dataflow.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;;GAKG;AAEH,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EACL,iBAAiB,EACjB,cAAc,GAGf,MAAM,kBAAkB,CAAC;AAG1B;;;;;;GAMG;AACH,MAAM,UAAU,aAAa;IAC3B,MAAM,WAAW,GAAG,IAAI,GAAG,EAA0B,CAAC;IACtD,MAAM,aAAa,GAAG,IAAI,GAAG,EAA6B,CAAC;IAE3D,MAAM,aAAa,GAAG,CAAC,QAAgB,EAAuB,EAAE;QAC9D,IAAI,KAAK,GAAG,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QACtC,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACxB,KAAK,GAAG,IAAI,cAAc,CAAC,IAAI,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC,CAAC;YACzD,WAAW,CAAC,GAAG,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;QACnC,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC,CAAC;IAEF,MAAM,eAAe,GAAG,CAAC,QAAgB,EAAwB,EAAE;QACjE,IAAI,YAAY,GAAG,aAAa,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAC/C,IAAI,YAAY,KAAK,SAAS,EAAE,CAAC;YAC/B,YAAY,GAAG,IAAI,iBAAiB,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC,CAAC;YAC9D,aAAa,CAAC,GAAG,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;QAC5C,CAAC;QACD,OAAO,YAAY,CAAC;IACtB,CAAC,CAAC;IAEF,OAAO,EAAE,eAAe,EAAE,aAAa,EAAE,CAAC;AAC5C,CAAC"}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Copyright (c) 2025 Elara AI Pty Ltd
|
|
3
|
+
* Licensed under BSL 1.1. See LICENSE for details.
|
|
4
|
+
*/
|
|
5
|
+
/**
|
|
6
|
+
* The gate every request to one repository passes, mounted on
|
|
7
|
+
* `/api/repos/:repo/*` ahead of the routes and after any auth, so an
|
|
8
|
+
* unauthorized request is answered 401 rather than 404. A refusal is JSON with
|
|
9
|
+
* its HTTP status, since a gate cannot know the BEAST2 type the route answers.
|
|
10
|
+
*
|
|
11
|
+
* Creating and removing a repository, at `/api/repos/:repo` itself, pass either
|
|
12
|
+
* gate, whatever order a host mounts them in: they act on a repository that may
|
|
13
|
+
* not exist yet, or is being removed, and their routes answer for it.
|
|
14
|
+
*/
|
|
15
|
+
import type { MiddlewareHandler } from 'hono';
|
|
16
|
+
import { type StorageBackend } from '@elaraai/e3-core';
|
|
17
|
+
/**
|
|
18
|
+
* The gate of a host's repositories: the repository exists, is not being
|
|
19
|
+
* removed, and is opened as every way into a repository opens it — upgraded in
|
|
20
|
+
* place when an older release wrote it, and refused, naming the fix, when this
|
|
21
|
+
* e3 cannot read it. A repository being removed answers its status alone, and
|
|
22
|
+
* is not opened. It goes through the storage backend's `RepoStore`, so a host
|
|
23
|
+
* mounts it over its own.
|
|
24
|
+
*
|
|
25
|
+
* @param storage - Storage backend, whose `RepoStore` keeps the repositories
|
|
26
|
+
* @param getRepoPath - A repository's identifier from its name, as the stores
|
|
27
|
+
* other than the `RepoStore` take it
|
|
28
|
+
* @returns The middleware
|
|
29
|
+
*/
|
|
30
|
+
export declare function createRepositoryGate(storage: StorageBackend, getRepoPath: (repo: string) => string): MiddlewareHandler;
|
|
31
|
+
/**
|
|
32
|
+
* The gate of a server of one repository: the repository is `default`, and a
|
|
33
|
+
* request to any other is not found. The server opens its repository as it
|
|
34
|
+
* starts.
|
|
35
|
+
*
|
|
36
|
+
* @returns The middleware
|
|
37
|
+
*/
|
|
38
|
+
export declare function createSingleRepositoryGate(): MiddlewareHandler;
|
|
39
|
+
//# sourceMappingURL=repository.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"repository.d.ts","sourceRoot":"","sources":["../../../src/middleware/repository.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;;;;;;GASG;AAEH,OAAO,KAAK,EAAW,iBAAiB,EAAE,MAAM,MAAM,CAAC;AACvD,OAAO,EAAkB,KAAK,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAQvE;;;;;;;;;;;;GAYG;AACH,wBAAgB,oBAAoB,CAClC,OAAO,EAAE,cAAc,EACvB,WAAW,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,MAAM,GACpC,iBAAiB,CAmCnB;AAED;;;;;;GAMG;AACH,wBAAgB,0BAA0B,IAAI,iBAAiB,CAU9D"}
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Copyright (c) 2025 Elara AI Pty Ltd
|
|
3
|
+
* Licensed under BSL 1.1. See LICENSE for details.
|
|
4
|
+
*/
|
|
5
|
+
import { repositoryOpen } from '@elaraai/e3-core';
|
|
6
|
+
import { sendJsonError } from '../errors.js';
|
|
7
|
+
/** Whether a request creates or removes a repository. */
|
|
8
|
+
function createsOrRemoves(c) {
|
|
9
|
+
return (c.req.method === 'PUT' || c.req.method === 'DELETE') && /\/repos\/[^/]+$/.test(c.req.path);
|
|
10
|
+
}
|
|
11
|
+
/**
|
|
12
|
+
* The gate of a host's repositories: the repository exists, is not being
|
|
13
|
+
* removed, and is opened as every way into a repository opens it — upgraded in
|
|
14
|
+
* place when an older release wrote it, and refused, naming the fix, when this
|
|
15
|
+
* e3 cannot read it. A repository being removed answers its status alone, and
|
|
16
|
+
* is not opened. It goes through the storage backend's `RepoStore`, so a host
|
|
17
|
+
* mounts it over its own.
|
|
18
|
+
*
|
|
19
|
+
* @param storage - Storage backend, whose `RepoStore` keeps the repositories
|
|
20
|
+
* @param getRepoPath - A repository's identifier from its name, as the stores
|
|
21
|
+
* other than the `RepoStore` take it
|
|
22
|
+
* @returns The middleware
|
|
23
|
+
*/
|
|
24
|
+
export function createRepositoryGate(storage, getRepoPath) {
|
|
25
|
+
return async (c, next) => {
|
|
26
|
+
if (createsOrRemoves(c)) {
|
|
27
|
+
await next();
|
|
28
|
+
return;
|
|
29
|
+
}
|
|
30
|
+
const repo = c.req.param('repo');
|
|
31
|
+
// A repository whose metadata this e3 cannot read is refused naming the
|
|
32
|
+
// fix, not with a bare 500.
|
|
33
|
+
let metadata;
|
|
34
|
+
try {
|
|
35
|
+
metadata = await storage.repos.getMetadata(repo);
|
|
36
|
+
}
|
|
37
|
+
catch (err) {
|
|
38
|
+
return sendJsonError(err);
|
|
39
|
+
}
|
|
40
|
+
if (!metadata) {
|
|
41
|
+
return c.json({ error: 'not_found', message: `Repository '${repo}' not found` }, 404);
|
|
42
|
+
}
|
|
43
|
+
if (metadata.status.type === 'deleting') {
|
|
44
|
+
if (!/\/repos\/[^/]+\/status$/.test(c.req.path)) {
|
|
45
|
+
return c.json({ error: 'not_found', message: `Repository '${repo}' not found` }, 404);
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
else {
|
|
49
|
+
try {
|
|
50
|
+
await repositoryOpen(storage, getRepoPath(repo));
|
|
51
|
+
}
|
|
52
|
+
catch (err) {
|
|
53
|
+
return sendJsonError(err);
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
await next();
|
|
57
|
+
};
|
|
58
|
+
}
|
|
59
|
+
/**
|
|
60
|
+
* The gate of a server of one repository: the repository is `default`, and a
|
|
61
|
+
* request to any other is not found. The server opens its repository as it
|
|
62
|
+
* starts.
|
|
63
|
+
*
|
|
64
|
+
* @returns The middleware
|
|
65
|
+
*/
|
|
66
|
+
export function createSingleRepositoryGate() {
|
|
67
|
+
return async (c, next) => {
|
|
68
|
+
if (!createsOrRemoves(c)) {
|
|
69
|
+
const repo = c.req.param('repo');
|
|
70
|
+
if (repo !== 'default') {
|
|
71
|
+
return c.json({ error: 'not_found', message: `Repository '${repo}' not found` }, 404);
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
await next();
|
|
75
|
+
};
|
|
76
|
+
}
|
|
77
|
+
//# sourceMappingURL=repository.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"repository.js","sourceRoot":"","sources":["../../../src/middleware/repository.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAcH,OAAO,EAAE,cAAc,EAAuB,MAAM,kBAAkB,CAAC;AACvE,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAE7C,yDAAyD;AACzD,SAAS,gBAAgB,CAAC,CAAU;IAClC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,KAAK,KAAK,IAAI,CAAC,CAAC,GAAG,CAAC,MAAM,KAAK,QAAQ,CAAC,IAAI,iBAAiB,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;AACrG,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,UAAU,oBAAoB,CAClC,OAAuB,EACvB,WAAqC;IAErC,OAAO,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,EAAE;QACvB,IAAI,gBAAgB,CAAC,CAAC,CAAC,EAAE,CAAC;YACxB,MAAM,IAAI,EAAE,CAAC;YACb,OAAO;QACT,CAAC;QAED,MAAM,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAE,CAAC;QAElC,wEAAwE;QACxE,4BAA4B;QAC5B,IAAI,QAAQ,CAAC;QACb,IAAI,CAAC;YACH,QAAQ,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;QACnD,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO,aAAa,CAAC,GAAG,CAAC,CAAC;QAC5B,CAAC;QACD,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,OAAO,CAAC,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,WAAW,EAAE,OAAO,EAAE,eAAe,IAAI,aAAa,EAAE,EAAE,GAAG,CAAC,CAAC;QACxF,CAAC;QAED,IAAI,QAAQ,CAAC,MAAM,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;YACxC,IAAI,CAAC,yBAAyB,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;gBAChD,OAAO,CAAC,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,WAAW,EAAE,OAAO,EAAE,eAAe,IAAI,aAAa,EAAE,EAAE,GAAG,CAAC,CAAC;YACxF,CAAC;QACH,CAAC;aAAM,CAAC;YACN,IAAI,CAAC;gBACH,MAAM,cAAc,CAAC,OAAO,EAAE,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC;YACnD,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,OAAO,aAAa,CAAC,GAAG,CAAC,CAAC;YAC5B,CAAC;QACH,CAAC;QAED,MAAM,IAAI,EAAE,CAAC;IACf,CAAC,CAAC;AACJ,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,0BAA0B;IACxC,OAAO,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,EAAE;QACvB,IAAI,CAAC,gBAAgB,CAAC,CAAC,CAAC,EAAE,CAAC;YACzB,MAAM,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;YACjC,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;gBACvB,OAAO,CAAC,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,WAAW,EAAE,OAAO,EAAE,eAAe,IAAI,aAAa,EAAE,EAAE,GAAG,CAAC,CAAC;YACxF,CAAC;QACH,CAAC;QACD,MAAM,IAAI,EAAE,CAAC;IACf,CAAC,CAAC;AACJ,CAAC"}
|
|
@@ -8,8 +8,8 @@ import { type StorageBackend, type TransferBackend } from '@elaraai/e3-core';
|
|
|
8
8
|
* Create generic upload/download data endpoints.
|
|
9
9
|
*
|
|
10
10
|
* Returns two Hono apps:
|
|
11
|
-
* - `uploads`: PUT /:id — upload
|
|
12
|
-
* PUT /:id/parts/:part — one part of a dataset upload
|
|
11
|
+
* - `uploads`: PUT /:id — upload a package zip;
|
|
12
|
+
* PUT /:id/parts/:part — one part of a dataset upload
|
|
13
13
|
* - `downloads`: GET /:id — download data (dataset BEAST2 or package zip)
|
|
14
14
|
*
|
|
15
15
|
* These are unauthenticated — the UUID in the URL is the sole capability.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../src/routes/data.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../src/routes/data.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAO5B,OAAO,EAKL,KAAK,cAAc,EACnB,KAAK,eAAe,EACrB,MAAM,kBAAkB,CAAC;AAiB1B;;;;;;;;;;GAUG;AACH,wBAAgB,mBAAmB,CACjC,eAAe,EAAE,eAAe,EAChC,OAAO,EAAE,cAAc,EACvB,WAAW,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,MAAM;;;EAgKtC"}
|
package/dist/src/routes/data.js
CHANGED
|
@@ -7,12 +7,9 @@ import { mkdir, open, writeFile, readFile, unlink } from 'node:fs/promises';
|
|
|
7
7
|
import { createWriteStream } from 'node:fs';
|
|
8
8
|
import { Readable, Transform } from 'node:stream';
|
|
9
9
|
import { pipeline } from 'node:stream/promises';
|
|
10
|
-
import { join } from 'node:path';
|
|
11
|
-
import { tmpdir } from 'node:os';
|
|
12
10
|
import { variant } from '@elaraai/east';
|
|
13
|
-
import { transferPartCount, transferPartRange } from '@elaraai/e3-types';
|
|
14
|
-
import {
|
|
15
|
-
const STAGING_DIR = join(tmpdir(), 'e3-transfers');
|
|
11
|
+
import { BEAST2_CONTENT_TYPE, transferPartCount, transferPartRange } from '@elaraai/e3-types';
|
|
12
|
+
import { ObjectNotFoundError, packageStagingPath, transferStagingDir, transferStagingPath, } from '@elaraai/e3-core';
|
|
16
13
|
/**
|
|
17
14
|
* Reject requests that carry an Authorization header.
|
|
18
15
|
*
|
|
@@ -31,8 +28,8 @@ function rejectAuthHeader(c) {
|
|
|
31
28
|
* Create generic upload/download data endpoints.
|
|
32
29
|
*
|
|
33
30
|
* Returns two Hono apps:
|
|
34
|
-
* - `uploads`: PUT /:id — upload
|
|
35
|
-
* PUT /:id/parts/:part — one part of a dataset upload
|
|
31
|
+
* - `uploads`: PUT /:id — upload a package zip;
|
|
32
|
+
* PUT /:id/parts/:part — one part of a dataset upload
|
|
36
33
|
* - `downloads`: GET /:id — download data (dataset BEAST2 or package zip)
|
|
37
34
|
*
|
|
38
35
|
* These are unauthenticated — the UUID in the URL is the sole capability.
|
|
@@ -41,40 +38,13 @@ function rejectAuthHeader(c) {
|
|
|
41
38
|
export function createDataEndpoints(transferBackend, storage, getRepoPath) {
|
|
42
39
|
const uploads = new Hono();
|
|
43
40
|
const downloads = new Hono();
|
|
44
|
-
// PUT /api/uploads/:id — Upload
|
|
41
|
+
// PUT /api/uploads/:id — Upload a package zip, staged for the trigger
|
|
42
|
+
// endpoint to process
|
|
45
43
|
uploads.put('/:id', async (c) => {
|
|
46
44
|
const rejected = rejectAuthHeader(c);
|
|
47
45
|
if (rejected)
|
|
48
46
|
return rejected;
|
|
49
47
|
const id = c.req.param('id');
|
|
50
|
-
// Try dataset upload first
|
|
51
|
-
const dsRecord = await transferBackend.datasetUpload.get(id);
|
|
52
|
-
if (dsRecord) {
|
|
53
|
-
// Streamed to the repo's own staging area: a delivery can be far larger
|
|
54
|
-
// than this process's heap, and the commit turns the staged file into an
|
|
55
|
-
// object by link or rename — which only works on the repo's device.
|
|
56
|
-
const repoPath = getRepoPath(dsRecord.repo);
|
|
57
|
-
const stagingPath = transferStagingPath(repoPath, id);
|
|
58
|
-
await mkdir(transferStagingDir(repoPath), { recursive: true });
|
|
59
|
-
const stream = c.req.raw.body;
|
|
60
|
-
try {
|
|
61
|
-
if (stream) {
|
|
62
|
-
await pipeline(Readable.fromWeb(stream), createWriteStream(stagingPath));
|
|
63
|
-
}
|
|
64
|
-
else {
|
|
65
|
-
await writeFile(stagingPath, new Uint8Array(await c.req.arrayBuffer()));
|
|
66
|
-
}
|
|
67
|
-
}
|
|
68
|
-
catch (err) {
|
|
69
|
-
// A broken upload leaves no partial file behind; gc sweeps the staging
|
|
70
|
-
// directory only as the backstop for a server that crashed mid-upload.
|
|
71
|
-
// The transfer record stays, so the client may retry the PUT.
|
|
72
|
-
await unlink(stagingPath).catch(() => { });
|
|
73
|
-
throw err;
|
|
74
|
-
}
|
|
75
|
-
return new Response(null, { status: 200 });
|
|
76
|
-
}
|
|
77
|
-
// Try package import — stage file for later processing by trigger endpoint
|
|
78
48
|
const pkgRecord = await transferBackend.packageImport.get(id);
|
|
79
49
|
if (pkgRecord) {
|
|
80
50
|
const body = new Uint8Array(await c.req.arrayBuffer());
|
|
@@ -82,16 +52,18 @@ export function createDataEndpoints(transferBackend, storage, getRepoPath) {
|
|
|
82
52
|
await transferBackend.packageImport.delete(id);
|
|
83
53
|
return new Response(`size mismatch: expected ${pkgRecord.size}, got ${body.byteLength}`, { status: 400 });
|
|
84
54
|
}
|
|
85
|
-
const
|
|
86
|
-
await mkdir(
|
|
87
|
-
await writeFile(
|
|
55
|
+
const repoPath = getRepoPath(pkgRecord.repo);
|
|
56
|
+
await mkdir(transferStagingDir(repoPath), { recursive: true });
|
|
57
|
+
await writeFile(packageStagingPath(repoPath, id), body);
|
|
88
58
|
await transferBackend.packageImport.updateStatus(id, variant('uploaded', null));
|
|
89
59
|
return new Response(null, { status: 200 });
|
|
90
60
|
}
|
|
91
61
|
return new Response('Not found', { status: 404 });
|
|
92
62
|
});
|
|
93
|
-
// PUT /api/uploads/:id/parts/:part — One part of a dataset upload
|
|
94
|
-
//
|
|
63
|
+
// PUT /api/uploads/:id/parts/:part — One part of a dataset upload. The staged
|
|
64
|
+
// file sits in the repository, far larger than this process's heap if need
|
|
65
|
+
// be, because the commit turns it into an object by link or rename, which
|
|
66
|
+
// only works on the repository's device.
|
|
95
67
|
uploads.put('/:id/parts/:part', async (c) => {
|
|
96
68
|
const rejected = rejectAuthHeader(c);
|
|
97
69
|
if (rejected)
|
|
@@ -102,6 +74,11 @@ export function createDataEndpoints(transferBackend, storage, getRepoPath) {
|
|
|
102
74
|
if (!record || partBytes === null) {
|
|
103
75
|
return new Response('Not found', { status: 404 });
|
|
104
76
|
}
|
|
77
|
+
// A part sent once the commit is under way could rewrite the bytes it
|
|
78
|
+
// verifies.
|
|
79
|
+
if ((await transferBackend.datasetUpload.getCommitStatus(id)) !== null) {
|
|
80
|
+
return new Response('the upload is committed: it takes no more parts', { status: 409 });
|
|
81
|
+
}
|
|
105
82
|
const part = Number(c.req.param('part'));
|
|
106
83
|
const range = transferPartRange(record.size, partBytes, part);
|
|
107
84
|
if (!range) {
|
|
@@ -109,9 +86,9 @@ export function createDataEndpoints(transferBackend, storage, getRepoPath) {
|
|
|
109
86
|
}
|
|
110
87
|
// Every part streams to its own offset in the one staged file — in any
|
|
111
88
|
// order, concurrently, and a re-sent part over the old one — so the commit
|
|
112
|
-
// adopts the file
|
|
113
|
-
//
|
|
114
|
-
//
|
|
89
|
+
// adopts the file with nothing to assemble. Opening with 'a' creates the
|
|
90
|
+
// file without truncating the parts already there; the write itself needs
|
|
91
|
+
// a positioned handle.
|
|
115
92
|
const repoPath = getRepoPath(record.repo);
|
|
116
93
|
const stagingPath = transferStagingPath(repoPath, id);
|
|
117
94
|
await mkdir(transferStagingDir(repoPath), { recursive: true });
|
|
@@ -178,7 +155,7 @@ export function createDataEndpoints(transferBackend, storage, getRepoPath) {
|
|
|
178
155
|
// Try package export
|
|
179
156
|
const pkgRecord = await transferBackend.packageExport.get(id);
|
|
180
157
|
if (pkgRecord && pkgRecord.status.type === 'completed') {
|
|
181
|
-
const stagingPath =
|
|
158
|
+
const stagingPath = packageStagingPath(getRepoPath(pkgRecord.repo), id);
|
|
182
159
|
try {
|
|
183
160
|
const fileData = await readFile(stagingPath);
|
|
184
161
|
await unlink(stagingPath).catch(() => { });
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"data.js","sourceRoot":"","sources":["../../../src/routes/data.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAC5B,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC5E,OAAO,EAAE,iBAAiB,EAAE,MAAM,SAAS,CAAC;AAC5C,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAClD,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAChD,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"data.js","sourceRoot":"","sources":["../../../src/routes/data.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAC5B,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC5E,OAAO,EAAE,iBAAiB,EAAE,MAAM,SAAS,CAAC;AAC5C,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAClD,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAChD,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AACxC,OAAO,EAAE,mBAAmB,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAC9F,OAAO,EACL,mBAAmB,EACnB,kBAAkB,EAClB,kBAAkB,EAClB,mBAAmB,GAGpB,MAAM,kBAAkB,CAAC;AAE1B;;;;;;;GAOG;AACH,SAAS,gBAAgB,CAAC,CAAwD;IAChF,IAAI,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,eAAe,CAAC,EAAE,CAAC;QAClC,OAAO,IAAI,QAAQ,CAAC,yDAAyD,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC;IAClG,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,UAAU,mBAAmB,CACjC,eAAgC,EAChC,OAAuB,EACvB,WAAqC;IAErC,MAAM,OAAO,GAAG,IAAI,IAAI,EAAE,CAAC;IAC3B,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC;IAE7B,sEAAsE;IACtE,sBAAsB;IACtB,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;QAC9B,MAAM,QAAQ,GAAG,gBAAgB,CAAC,CAAC,CAAC,CAAC;QACrC,IAAI,QAAQ;YAAE,OAAO,QAAQ,CAAC;QAE9B,MAAM,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAE,CAAC;QAC9B,MAAM,SAAS,GAAG,MAAM,eAAe,CAAC,aAAa,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC9D,IAAI,SAAS,EAAE,CAAC;YACd,MAAM,IAAI,GAAG,IAAI,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC,CAAC;YACvD,IAAI,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,SAAS,CAAC,IAAI,EAAE,CAAC;gBAC/C,MAAM,eAAe,CAAC,aAAa,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;gBAC/C,OAAO,IAAI,QAAQ,CACjB,2BAA2B,SAAS,CAAC,IAAI,SAAS,IAAI,CAAC,UAAU,EAAE,EACnE,EAAE,MAAM,EAAE,GAAG,EAAE,CAChB,CAAC;YACJ,CAAC;YAED,MAAM,QAAQ,GAAG,WAAW,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;YAC7C,MAAM,KAAK,CAAC,kBAAkB,CAAC,QAAQ,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YAC/D,MAAM,SAAS,CAAC,kBAAkB,CAAC,QAAQ,EAAE,EAAE,CAAC,EAAE,IAAI,CAAC,CAAC;YACxD,MAAM,eAAe,CAAC,aAAa,CAAC,YAAY,CAAC,EAAE,EAAE,OAAO,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC,CAAC;YAEhF,OAAO,IAAI,QAAQ,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC;QAC7C,CAAC;QAED,OAAO,IAAI,QAAQ,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC;IACpD,CAAC,CAAC,CAAC;IAEH,8EAA8E;IAC9E,2EAA2E;IAC3E,0EAA0E;IAC1E,yCAAyC;IACzC,OAAO,CAAC,GAAG,CAAC,kBAAkB,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;QAC1C,MAAM,QAAQ,GAAG,gBAAgB,CAAC,CAAC,CAAC,CAAC;QACrC,IAAI,QAAQ;YAAE,OAAO,QAAQ,CAAC;QAE9B,MAAM,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAE,CAAC;QAC9B,MAAM,MAAM,GAAG,MAAM,eAAe,CAAC,aAAa,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC3D,MAAM,SAAS,GAAG,MAAM,CAAC,CAAC,CAAC,MAAM,eAAe,CAAC,aAAa,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QACvF,IAAI,CAAC,MAAM,IAAI,SAAS,KAAK,IAAI,EAAE,CAAC;YAClC,OAAO,IAAI,QAAQ,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC;QACpD,CAAC;QACD,sEAAsE;QACtE,YAAY;QACZ,IAAI,CAAC,MAAM,eAAe,CAAC,aAAa,CAAC,eAAe,CAAC,EAAE,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YACvE,OAAO,IAAI,QAAQ,CAAC,iDAAiD,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC;QAC1F,CAAC;QACD,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;QACzC,MAAM,KAAK,GAAG,iBAAiB,CAAC,MAAM,CAAC,IAAI,EAAE,SAAS,EAAE,IAAI,CAAC,CAAC;QAC9D,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,OAAO,IAAI,QAAQ,CACjB,WAAW,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,oBAAoB,iBAAiB,CAAC,MAAM,CAAC,IAAI,EAAE,SAAS,CAAC,QAAQ,EACnG,EAAE,MAAM,EAAE,GAAG,EAAE,CAChB,CAAC;QACJ,CAAC;QAED,uEAAuE;QACvE,2EAA2E;QAC3E,yEAAyE;QACzE,0EAA0E;QAC1E,uBAAuB;QACvB,MAAM,QAAQ,GAAG,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAC1C,MAAM,WAAW,GAAG,mBAAmB,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;QACtD,MAAM,KAAK,CAAC,kBAAkB,CAAC,QAAQ,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC/D,MAAM,CAAC,MAAM,IAAI,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC;QAE7C,qEAAqE;QACrE,sEAAsE;QACtE,MAAM,QAAQ,GAAG,KAAK,CAAC,GAAG,GAAG,KAAK,CAAC,KAAK,CAAC;QACzC,IAAI,QAAQ,GAAG,CAAC,CAAC;QACjB,MAAM,OAAO,GAAG,IAAI,SAAS,CAAC;YAC5B,SAAS,CAAC,KAAa,EAAE,SAAS,EAAE,QAAQ;gBAC1C,QAAQ,IAAI,KAAK,CAAC,MAAM,CAAC;gBACzB,IAAI,QAAQ,GAAG,QAAQ,EAAE,CAAC;oBACxB,QAAQ,CAAC,IAAI,KAAK,CAAC,QAAQ,IAAI,OAAO,QAAQ,4BAA4B,CAAC,CAAC,CAAC;gBAC/E,CAAC;qBAAM,CAAC;oBACN,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;gBACxB,CAAC;YACH,CAAC;SACF,CAAC,CAAC;QACH,MAAM,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC;QAC5B,IAAI,CAAC;YACH,MAAM,QAAQ,CACZ,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,IAA8C,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC,EAC3F,OAAO,EACP,iBAAiB,CAAC,WAAW,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC,CACpE,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,QAAQ,GAAG,QAAQ,EAAE,CAAC;gBACxB,OAAO,IAAI,QAAQ,CAAC,QAAQ,IAAI,OAAO,QAAQ,4BAA4B,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC;YAChG,CAAC;YACD,MAAM,GAAG,CAAC;QACZ,CAAC;QACD,IAAI,QAAQ,KAAK,QAAQ,EAAE,CAAC;YAC1B,OAAO,IAAI,QAAQ,CAAC,QAAQ,IAAI,OAAO,QAAQ,eAAe,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC;QAC7F,CAAC;QACD,OAAO,IAAI,QAAQ,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC;IAC7C,CAAC,CAAC,CAAC;IAEH,yEAAyE;IACzE,SAAS,CAAC,GAAG,CAAC,MAAM,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;QAChC,MAAM,QAAQ,GAAG,gBAAgB,CAAC,CAAC,CAAC,CAAC;QACrC,IAAI,QAAQ;YAAE,OAAO,QAAQ,CAAC;QAE9B,MAAM,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAE,CAAC;QAE9B,uBAAuB;QACvB,MAAM,QAAQ,GAAG,MAAM,eAAe,CAAC,eAAe,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC/D,IAAI,QAAQ,EAAE,CAAC;YACb,MAAM,QAAQ,GAAG,WAAW,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;YAC5C,IAAI,CAAC;gBACH,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC;gBACjE,MAAM,eAAe,CAAC,eAAe,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;gBACjD,OAAO,IAAI,QAAQ,CAAC,IAAI,EAAE;oBACxB,OAAO,EAAE;wBACP,cAAc,EAAE,mBAAmB;wBACnC,gBAAgB,EAAE,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC;wBACzC,kBAAkB,EAAE,QAAQ,CAAC,IAAI;qBAClC;iBACF,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,eAAe,CAAC,eAAe,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;gBACjD,IAAI,GAAG,YAAY,mBAAmB,EAAE,CAAC;oBACvC,OAAO,IAAI,QAAQ,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC;gBACpD,CAAC;gBACD,MAAM,GAAG,CAAC;YACZ,CAAC;QACH,CAAC;QAED,qBAAqB;QACrB,MAAM,SAAS,GAAG,MAAM,eAAe,CAAC,aAAa,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC9D,IAAI,SAAS,IAAI,SAAS,CAAC,MAAM,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;YACvD,MAAM,WAAW,GAAG,kBAAkB,CAAC,WAAW,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;YACxE,IAAI,CAAC;gBACH,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAC,WAAW,CAAC,CAAC;gBAC7C,MAAM,MAAM,CAAC,WAAW,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;gBAC1C,MAAM,eAAe,CAAC,aAAa,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;gBAE/C,OAAO,IAAI,QAAQ,CAAC,QAAQ,EAAE;oBAC5B,MAAM,EAAE,GAAG;oBACX,OAAO,EAAE;wBACP,cAAc,EAAE,iBAAiB;wBACjC,gBAAgB,EAAE,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC;qBAC9C;iBACF,CAAC,CAAC;YACL,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO,IAAI,QAAQ,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC;YACpD,CAAC;QACH,CAAC;QAED,OAAO,IAAI,QAAQ,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC;IACpD,CAAC,CAAC,CAAC;IAEH,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC;AAChC,CAAC"}
|
|
@@ -10,9 +10,6 @@ export interface DatasetRouteOptions {
|
|
|
10
10
|
* 4 MiB; deployments with tighter response limits (e.g. Lambda proxy's
|
|
11
11
|
* 6 MB, base64-inflated) pass a smaller budget. */
|
|
12
12
|
pageByteBudget?: number;
|
|
13
|
-
/** Absolute blob-buffering cap for the page endpoint. Defaults to
|
|
14
|
-
* 512 MiB, until range reads land. */
|
|
15
|
-
pageReadMaxBytes?: number;
|
|
16
13
|
}
|
|
17
14
|
export declare function createDatasetRoutes(storage: StorageBackend, getRepoPath: (repo: string) => string, transferBackend?: TransferBackend, options?: DatasetRouteOptions): Hono<import("hono/types").BlankEnv, import("hono/types").BlankSchema, "/">;
|
|
18
15
|
//# sourceMappingURL=datasets.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"datasets.d.ts","sourceRoot":"","sources":["../../../src/routes/datasets.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAE5B,OAAO,KAAK,EAAE,cAAc,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAaxE,+CAA+C;AAC/C,MAAM,WAAW,mBAAmB;IAClC;;wDAEoD;IACpD,cAAc,CAAC,EAAE,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"datasets.d.ts","sourceRoot":"","sources":["../../../src/routes/datasets.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAE5B,OAAO,KAAK,EAAE,cAAc,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAaxE,+CAA+C;AAC/C,MAAM,WAAW,mBAAmB;IAClC;;wDAEoD;IACpD,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AASD,wBAAgB,mBAAmB,CACjC,OAAO,EAAE,cAAc,EACvB,WAAW,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,MAAM,EACrC,eAAe,CAAC,EAAE,eAAe,EACjC,OAAO,CAAC,EAAE,mBAAmB,8EA6G9B"}
|
|
@@ -66,13 +66,19 @@ export function createDatasetRoutes(storage, getRepoPath, transferBackend, optio
|
|
|
66
66
|
const key = c.req.query('key');
|
|
67
67
|
const prefix = c.req.query('prefix');
|
|
68
68
|
const fields = c.req.queries('field');
|
|
69
|
+
// `from` / `to` repeat, as `field` does: each is one `.east` literal of
|
|
70
|
+
// a leading prefix of the flattened key.
|
|
71
|
+
const from = c.req.queries('from');
|
|
72
|
+
const to = c.req.queries('to');
|
|
73
|
+
const index = c.req.query('index');
|
|
69
74
|
return findDatasetKey(storage, repoPath, ws, treePath, {
|
|
70
75
|
...(key !== undefined && { key }),
|
|
71
76
|
...(prefix !== undefined && { prefix }),
|
|
72
77
|
...(fields !== undefined && fields.length > 0 && { fields }),
|
|
78
|
+
...(from !== undefined && from.length > 0 && { from }),
|
|
79
|
+
...(to !== undefined && to.length > 0 && { to }),
|
|
80
|
+
...(index !== undefined && index !== '' && { index }),
|
|
73
81
|
...(hash !== undefined && hash !== '' && { hash }),
|
|
74
|
-
}, {
|
|
75
|
-
...(options?.pageReadMaxBytes !== undefined && { readMaxBytes: options.pageReadMaxBytes }),
|
|
76
82
|
});
|
|
77
83
|
}
|
|
78
84
|
if (page) {
|
|
@@ -81,14 +87,17 @@ export function createDatasetRoutes(storage, getRepoPath, transferBackend, optio
|
|
|
81
87
|
...(intParam(c.req.query('offset')) !== undefined && { offset: intParam(c.req.query('offset')) }),
|
|
82
88
|
...(intParam(c.req.query('limit')) !== undefined && { limit: intParam(c.req.query('limit')) }),
|
|
83
89
|
...(intParam(c.req.query('segment')) !== undefined && { segment: intParam(c.req.query('segment')) }),
|
|
90
|
+
...(c.req.query('index') !== undefined && c.req.query('index') !== '' && { index: c.req.query('index') }),
|
|
91
|
+
...(c.req.query('join') === 'true' && { join: true }),
|
|
84
92
|
...(hash !== undefined && hash !== '' && { hash }),
|
|
85
93
|
};
|
|
86
94
|
return getDatasetPage(storage, repoPath, ws, treePath, window, {
|
|
87
95
|
...(options?.pageByteBudget !== undefined && { byteBudget: options.pageByteBudget }),
|
|
88
|
-
...(options?.pageReadMaxBytes !== undefined && { readMaxBytes: options.pageReadMaxBytes }),
|
|
89
96
|
});
|
|
90
97
|
}
|
|
91
|
-
|
|
98
|
+
// `segments=true`: a collection held as a segment manifest is answered
|
|
99
|
+
// with its manifest's hash, for a client that downloads the segments.
|
|
100
|
+
return getDataset(storage, repoPath, ws, treePath, repo, c.req.url, transferBackend, c.req.query('segments') === 'true');
|
|
92
101
|
});
|
|
93
102
|
// PUT /api/repos/:repo/workspaces/:ws/datasets/* - Set dataset value
|
|
94
103
|
app.put('/*', async (c) => {
|
|
@@ -100,11 +109,34 @@ export function createDatasetRoutes(storage, getRepoPath, transferBackend, optio
|
|
|
100
109
|
const datasetsPrefix = `/api/repos/${encodeURIComponent(repo)}/workspaces/${encodeURIComponent(ws)}/datasets/`;
|
|
101
110
|
const pathStr = fullPath.startsWith(datasetsPrefix) ? fullPath.slice(datasetsPrefix.length) : '';
|
|
102
111
|
const treePath = urlPathToTreePath(pathStr);
|
|
103
|
-
// Body is raw BEAST2
|
|
104
|
-
|
|
105
|
-
const body = new Uint8Array(buffer);
|
|
106
|
-
return setDataset(storage, repoPath, ws, treePath, body);
|
|
112
|
+
// Body is raw BEAST2, read as it arrives.
|
|
113
|
+
return setDataset(storage, repoPath, ws, treePath, bodyChunks(c.req.raw.body));
|
|
107
114
|
});
|
|
108
115
|
return app;
|
|
109
116
|
}
|
|
117
|
+
/**
|
|
118
|
+
* A request body's chunks, as they arrive. A reader that stops early — a
|
|
119
|
+
* refused upload — cancels the body, so the connection is not left holding it.
|
|
120
|
+
*/
|
|
121
|
+
async function* bodyChunks(body) {
|
|
122
|
+
if (body === null)
|
|
123
|
+
return;
|
|
124
|
+
const reader = body.getReader();
|
|
125
|
+
let done = false;
|
|
126
|
+
try {
|
|
127
|
+
for (;;) {
|
|
128
|
+
const next = await reader.read();
|
|
129
|
+
if (next.done) {
|
|
130
|
+
done = true;
|
|
131
|
+
return;
|
|
132
|
+
}
|
|
133
|
+
yield next.value;
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
finally {
|
|
137
|
+
if (!done)
|
|
138
|
+
await reader.cancel().catch(() => { });
|
|
139
|
+
reader.releaseLock();
|
|
140
|
+
}
|
|
141
|
+
}
|
|
110
142
|
//# sourceMappingURL=datasets.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"datasets.js","sourceRoot":"","sources":["../../../src/routes/datasets.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAC5B,OAAO,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAEtD,OAAO,EACL,YAAY,EACZ,qBAAqB,EACrB,0BAA0B,EAC1B,sBAAsB,EACtB,cAAc,EACd,UAAU,EACV,cAAc,EACd,gBAAgB,EAChB,UAAU,GACX,MAAM,yBAAyB,CAAC;
|
|
1
|
+
{"version":3,"file":"datasets.js","sourceRoot":"","sources":["../../../src/routes/datasets.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAC5B,OAAO,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAEtD,OAAO,EACL,YAAY,EACZ,qBAAqB,EACrB,0BAA0B,EAC1B,sBAAsB,EACtB,cAAc,EACd,UAAU,EACV,cAAc,EACd,gBAAgB,EAChB,UAAU,GACX,MAAM,yBAAyB,CAAC;AAUjC;sDACsD;AACtD,SAAS,QAAQ,CAAC,KAAyB;IACzC,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,EAAE;QAAE,OAAO,SAAS,CAAC;IAC1D,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC;AACvB,CAAC;AAED,MAAM,UAAU,mBAAmB,CACjC,OAAuB,EACvB,WAAqC,EACrC,eAAiC,EACjC,OAA6B;IAE7B,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;IAEvB,kEAAkE;IAClE,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;QACvB,MAAM,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAE,CAAC;QAClC,MAAM,QAAQ,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;QACnC,MAAM,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAE,CAAC;QAE9B,MAAM,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,MAAM,CAAC;QAC5C,MAAM,SAAS,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,WAAW,CAAC,KAAK,MAAM,CAAC;QACtD,MAAM,MAAM,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,MAAM,CAAC;QAEhD,IAAI,SAAS,IAAI,CAAC,IAAI,EAAE,CAAC;YACvB,OAAO,CAAC,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,8BAA8B,EAAE,EAAE,GAAG,CAAC,CAAC;QAChE,CAAC;QAED,IAAI,IAAI,IAAI,SAAS,IAAI,MAAM;YAAE,OAAO,qBAAqB,CAAC,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC;QACzF,IAAI,IAAI,IAAI,SAAS;YAAY,OAAO,0BAA0B,CAAC,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC;QAC9F,IAAI,IAAI,IAAI,MAAM;YAAe,OAAO,sBAAsB,CAAC,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC;QAE1F,OAAO,YAAY,CAAC,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC;IACjD,CAAC,CAAC,CAAC;IAEH,2EAA2E;IAC3E,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;QACxB,MAAM,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAE,CAAC;QAClC,MAAM,QAAQ,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;QACnC,MAAM,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAE,CAAC;QAE9B,4DAA4D;QAC5D,MAAM,QAAQ,GAAG,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC;QAC5B,MAAM,cAAc,GAAG,cAAc,kBAAkB,CAAC,IAAI,CAAC,eAAe,kBAAkB,CAAC,EAAE,CAAC,YAAY,CAAC;QAC/G,MAAM,OAAO,GAAG,QAAQ,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACjG,MAAM,QAAQ,GAAG,iBAAiB,CAAC,OAAO,CAAC,CAAC;QAE5C,MAAM,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,MAAM,CAAC;QAC5C,MAAM,SAAS,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,WAAW,CAAC,KAAK,MAAM,CAAC;QACtD,MAAM,MAAM,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,MAAM,CAAC;QAChD,MAAM,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,MAAM,CAAC;QAC5C,MAAM,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,MAAM,CAAC;QAE5C,IAAI,SAAS,IAAI,CAAC,IAAI,EAAE,CAAC;YACvB,OAAO,CAAC,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,8BAA8B,EAAE,EAAE,GAAG,CAAC,CAAC;QAChE,CAAC;QAED,IAAI,IAAI,IAAI,SAAS,IAAI,MAAM;YAAE,OAAO,qBAAqB,CAAC,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,QAAQ,CAAC,CAAC;QAC/F,IAAI,IAAI,IAAI,SAAS;YAAY,OAAO,0BAA0B,CAAC,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,QAAQ,CAAC,CAAC;QACpG,IAAI,IAAI,IAAI,MAAM;YAAe,OAAO,sBAAsB,CAAC,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,QAAQ,CAAC,CAAC;QAChG,IAAI,IAAI;YAAyB,OAAO,YAAY,CAAC,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,QAAQ,CAAC,CAAC;QACtF,IAAI,MAAM;YAAuB,OAAO,gBAAgB,CAAC,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,QAAQ,CAAC,CAAC;QAC1F,IAAI,IAAI,EAAE,CAAC;YACT,MAAM,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;YACjC,MAAM,GAAG,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAC/B,MAAM,MAAM,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;YACrC,MAAM,MAAM,GAAG,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;YACtC,wEAAwE;YACxE,yCAAyC;YACzC,MAAM,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;YACnC,MAAM,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YAC/B,MAAM,KAAK,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YACnC,OAAO,cAAc,CAAC,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,QAAQ,EAAE;gBACrD,GAAG,CAAC,GAAG,KAAK,SAAS,IAAI,EAAE,GAAG,EAAE,CAAC;gBACjC,GAAG,CAAC,MAAM,KAAK,SAAS,IAAI,EAAE,MAAM,EAAE,CAAC;gBACvC,GAAG,CAAC,MAAM,KAAK,SAAS,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC;gBAC5D,GAAG,CAAC,IAAI,KAAK,SAAS,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC;gBACtD,GAAG,CAAC,EAAE,KAAK,SAAS,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,EAAE,EAAE,CAAC;gBAChD,GAAG,CAAC,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;gBACrD,GAAG,CAAC,IAAI,KAAK,SAAS,IAAI,IAAI,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;aACnD,CAAC,CAAC;QACL,CAAC;QACD,IAAI,IAAI,EAAE,CAAC;YACT,MAAM,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;YACjC,MAAM,MAAM,GAAG;gBACb,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,KAAK,SAAS,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAE,EAAE,CAAC;gBAClG,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,KAAK,SAAS,IAAI,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,CAAE,EAAE,CAAC;gBAC/F,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,KAAK,SAAS,IAAI,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC,CAAE,EAAE,CAAC;gBACrG,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,SAAS,IAAI,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAE,EAAE,CAAC;gBAC1G,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,MAAM,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;gBACrD,GAAG,CAAC,IAAI,KAAK,SAAS,IAAI,IAAI,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;aACnD,CAAC;YACF,OAAO,cAAc,CAAC,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE;gBAC7D,GAAG,CAAC,OAAO,EAAE,cAAc,KAAK,SAAS,IAAI,EAAE,UAAU,EAAE,OAAO,CAAC,cAAc,EAAE,CAAC;aACrF,CAAC,CAAC;QACL,CAAC;QAED,uEAAuE;QACvE,sEAAsE;QACtE,OAAO,UAAU,CAAC,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,GAAG,EAAE,eAAe,EAAE,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,UAAU,CAAC,KAAK,MAAM,CAAC,CAAC;IAC3H,CAAC,CAAC,CAAC;IAEH,qEAAqE;IACrE,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;QACxB,MAAM,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAE,CAAC;QAClC,MAAM,QAAQ,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;QACnC,MAAM,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAE,CAAC;QAE9B,4DAA4D;QAC5D,MAAM,QAAQ,GAAG,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC;QAC5B,MAAM,cAAc,GAAG,cAAc,kBAAkB,CAAC,IAAI,CAAC,eAAe,kBAAkB,CAAC,EAAE,CAAC,YAAY,CAAC;QAC/G,MAAM,OAAO,GAAG,QAAQ,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACjG,MAAM,QAAQ,GAAG,iBAAiB,CAAC,OAAO,CAAC,CAAC;QAE5C,0CAA0C;QAC1C,OAAO,UAAU,CAAC,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,QAAQ,EAAE,UAAU,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;IACjF,CAAC,CAAC,CAAC;IAEH,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;GAGG;AACH,KAAK,SAAS,CAAC,CAAC,UAAU,CAAC,IAAuC;IAChE,IAAI,IAAI,KAAK,IAAI;QAAE,OAAO;IAC1B,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;IAChC,IAAI,IAAI,GAAG,KAAK,CAAC;IACjB,IAAI,CAAC;QACH,SAAS,CAAC;YACR,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;YACjC,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;gBACd,IAAI,GAAG,IAAI,CAAC;gBACZ,OAAO;YACT,CAAC;YACD,MAAM,IAAI,CAAC,KAAK,CAAC;QACnB,CAAC;IACH,CAAC;YAAS,CAAC;QACT,IAAI,CAAC,IAAI;YAAE,MAAM,MAAM,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAAyC,CAAC,CAAC,CAAC;QACxF,MAAM,CAAC,WAAW,EAAE,CAAC;IACvB,CAAC;AACH,CAAC"}
|
|
@@ -3,6 +3,41 @@
|
|
|
3
3
|
* Licensed under BSL 1.1. See LICENSE for details.
|
|
4
4
|
*/
|
|
5
5
|
import { Hono } from 'hono';
|
|
6
|
-
import {
|
|
7
|
-
|
|
6
|
+
import type { Budget, DataflowOrchestrator, ExecutionStateStore, StorageBackend } from '@elaraai/e3-core';
|
|
7
|
+
import type { GetRunner } from './functions.js';
|
|
8
|
+
/**
|
|
9
|
+
* The seams the dataflow routes go through, which the host that mounts them
|
|
10
|
+
* gives: what runs a repository's dataflows, and where their runs are kept.
|
|
11
|
+
* A local server's are a LocalOrchestrator and a FileStateStore for each
|
|
12
|
+
* repository; a cloud's run its dataflows on its own compute, and keep their
|
|
13
|
+
* state in its own store.
|
|
14
|
+
*/
|
|
15
|
+
export interface DataflowSeams {
|
|
16
|
+
/** The runner a repository's tasks and units run on, which the status also
|
|
17
|
+
* asks whether an execution recorded running can still finish */
|
|
18
|
+
getRunner: GetRunner;
|
|
19
|
+
/** The orchestrator that runs a repository's dataflows, and cancels one */
|
|
20
|
+
getOrchestrator: (repoPath: string) => DataflowOrchestrator;
|
|
21
|
+
/** The store a repository's runs keep their state in, which a poll and a
|
|
22
|
+
* cancel read: the one the orchestrator writes */
|
|
23
|
+
getStateStore: (repoPath: string) => ExecutionStateStore;
|
|
24
|
+
/** The tasks and units the loop keeps in flight; the orchestrator's own
|
|
25
|
+
* default when absent */
|
|
26
|
+
width?: number;
|
|
27
|
+
/** The budget the runner holds, which the poll and the budget route serve;
|
|
28
|
+
* absent for a host whose runners hold none */
|
|
29
|
+
budget?: Budget;
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* The routes of a workspace's dataflow: start, poll, cancel, its graph, its
|
|
33
|
+
* tasks' logs, and the budget a run gets.
|
|
34
|
+
*
|
|
35
|
+
* @param storage - Storage backend
|
|
36
|
+
* @param getRepoPath - A repository's path from its name
|
|
37
|
+
* @param seams - What runs a repository's dataflows, and where their runs are
|
|
38
|
+
* kept: a poll or a cancel reads the run's state from the store, whichever
|
|
39
|
+
* instance answers it
|
|
40
|
+
* @returns The routes
|
|
41
|
+
*/
|
|
42
|
+
export declare function createExecutionRoutes(storage: StorageBackend, getRepoPath: (repo: string) => string, seams: DataflowSeams): Hono<import("hono/types").BlankEnv, import("hono/types").BlankSchema, "/">;
|
|
8
43
|
//# sourceMappingURL=executions.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"executions.d.ts","sourceRoot":"","sources":["../../../src/routes/executions.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAC5B,OAAO,
|
|
1
|
+
{"version":3,"file":"executions.d.ts","sourceRoot":"","sources":["../../../src/routes/executions.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAC5B,OAAO,KAAK,EAAE,MAAM,EAAE,oBAAoB,EAAE,mBAAmB,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAY1G,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAEhD;;;;;;GAMG;AACH,MAAM,WAAW,aAAa;IAC5B;sEACkE;IAClE,SAAS,EAAE,SAAS,CAAC;IACrB,2EAA2E;IAC3E,eAAe,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,oBAAoB,CAAC;IAC5D;uDACmD;IACnD,aAAa,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,mBAAmB,CAAC;IACzD;8BAC0B;IAC1B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;oDACgD;IAChD,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,qBAAqB,CACnC,OAAO,EAAE,cAAc,EACvB,WAAW,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,MAAM,EACrC,KAAK,EAAE,aAAa,8EA+ErB"}
|