@ailuntz/resource-agent 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +49 -0
- package/bin/ailuntz-downloads.mjs +20 -0
- package/package.json +29 -0
- package/src/channel-controller.mjs +63 -0
- package/src/channel-policy.mjs +39 -0
- package/src/client.mjs +15 -0
- package/src/server.mjs +54 -0
- package/src/service.mjs +122 -0
package/README.md
ADDED
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
# @ailuntz/resource-agent
|
|
2
|
+
|
|
3
|
+
Authenticated short-lived S3 download links, with health-aware Cloudflare Tunnel selection and elastic connector workers. Clients receive one link; object data travels through the selected tunnel, without passing through the application backend.
|
|
4
|
+
|
|
5
|
+
```sh
|
|
6
|
+
npm install @ailuntz/resource-agent@1.1.0
|
|
7
|
+
```
|
|
8
|
+
|
|
9
|
+
Backend integration:
|
|
10
|
+
|
|
11
|
+
```js
|
|
12
|
+
import { signedRequest } from '@ailuntz/resource-agent/client';
|
|
13
|
+
const request = await signedRequest({
|
|
14
|
+
clientId: 'shop', secret: process.env.DOWNLOAD_CLIENT_SECRET,
|
|
15
|
+
path: '/v1/downloads',
|
|
16
|
+
body: { bucket: 'shop-files', key: 'resources/example.zip',
|
|
17
|
+
filename: 'example.zip', expiresAt: Date.now() + 120000 },
|
|
18
|
+
});
|
|
19
|
+
const result = await fetch(process.env.DOWNLOAD_AGENT_URL + '/v1/downloads', request);
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
Authenticate the user and check resource access on your backend **before** making this request. Never distribute the client secret or storage signing key to browsers.
|
|
23
|
+
|
|
24
|
+
## Runtime
|
|
25
|
+
|
|
26
|
+
Run `ailuntz-downloads serve --runtime /runtime` and `ailuntz-downloads channels --runtime /runtime` as separate supervised services. There is no native service installer.
|
|
27
|
+
|
|
28
|
+
- `secrets/agent.json`: S3 read-only signing credentials and per-client shared secrets/bucket allowlists.
|
|
29
|
+
- `channels.json`: four pre-provisioned tunnel definitions; first channel id `primary`.
|
|
30
|
+
- `channel-policy.json`: minimum/maximum channels, scaling and draining intervals.
|
|
31
|
+
- `state/`: shared writable health snapshots and admission markers.
|
|
32
|
+
- `AILUNTZ_CONFIG_DIR`: optional directory holding channel definitions/policy.
|
|
33
|
+
- `CLOUDFLARED_BIN`: path to a pinned cloudflared executable. In containers, `AILUNTZ_CONTAINER=1` sends worker output to container logs.
|
|
34
|
+
|
|
35
|
+
The primary tunnel and control tunnel run separately. The channel manager starts/stops additional cloudflared processes in its own container, without host process control or a Docker socket. Shared networking gives these workers access to the download gateway; shared state lets the gateway stop issuing new links before a lane drains.
|
|
36
|
+
|
|
37
|
+
## HTTP API
|
|
38
|
+
|
|
39
|
+
| Route | Purpose |
|
|
40
|
+
|---|---|
|
|
41
|
+
| `GET /healthz` | Liveness only |
|
|
42
|
+
| `POST /v1/downloads` | Signed GET link; maximum 120 seconds, bounded by authorization expiry |
|
|
43
|
+
| `POST /v1/objects/stat` | Authorized object size lookup |
|
|
44
|
+
|
|
45
|
+
`serve` binds to loopback port 19081. Signatures bind the caller, method, path, timestamp, nonce and body. Nonces cannot be reused within the process. Requests with a browser Origin, disallowed bucket, invalid path or expired grant are rejected. Restarting clears replay memory; the original authorization deadline still applies.
|
|
46
|
+
|
|
47
|
+
Scaling preserves existing transfers: stop issuing links, wait at least 150 seconds, and require zero observed in-flight requests before stopping a worker. Missing metrics never permit draining. New requests try healthy lanes after a failure; an interrupted download needs a new URL and a client that supports Range resumption. Multiple tunnels share the same physical uplink.
|
|
48
|
+
|
|
49
|
+
The package contains code only: no deployment credentials, tunnel tokens, customer objects or production host configuration. The Docker image `ailuntz/download-agent:1.1.0` installs this exact npm release and a pinned official cloudflared binary.
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import fs from 'node:fs';
|
|
3
|
+
import os from 'node:os';
|
|
4
|
+
import path from 'node:path';
|
|
5
|
+
const command = process.argv[2], index = process.argv.indexOf('--runtime');
|
|
6
|
+
const runtime = path.resolve(index >= 0 ? process.argv[index + 1] : process.env.AILUNTZ_RUNTIME || path.join(os.homedir(), '.ailuntz-downloads'));
|
|
7
|
+
process.env.AILUNTZ_RUNTIME = runtime;
|
|
8
|
+
try {
|
|
9
|
+
if (command === 'serve') await (await import('../src/server.mjs')).serve(runtime);
|
|
10
|
+
else if (command === 'channels') await import('../src/channel-controller.mjs');
|
|
11
|
+
else if (command === 'status') {
|
|
12
|
+
const state = JSON.parse(fs.readFileSync(runtime + '/state/channels.json'));
|
|
13
|
+
const agent = JSON.parse(fs.readFileSync(runtime + '/state/agent.json'));
|
|
14
|
+
console.log(JSON.stringify({agent, policy: state.policy, checkedAt: state.checkedAt, channels: state.channels}, null, 2));
|
|
15
|
+
if (Date.now() - Date.parse(agent.checkedAt) > 15000 || Date.now() - Date.parse(state.checkedAt) > 15000) process.exitCode = 1;
|
|
16
|
+
} else console.log('ailuntz-downloads <serve|channels|status> [--runtime directory]');
|
|
17
|
+
} catch {
|
|
18
|
+
console.error('Download service could not complete the operation; check local configuration and service status.');
|
|
19
|
+
process.exitCode = 1;
|
|
20
|
+
}
|
package/package.json
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@ailuntz/resource-agent",
|
|
3
|
+
"version": "1.1.0",
|
|
4
|
+
"description": "Authenticated S3 download links and elastic Cloudflare tunnel workers",
|
|
5
|
+
"type": "module",
|
|
6
|
+
"engines": {
|
|
7
|
+
"node": ">=22"
|
|
8
|
+
},
|
|
9
|
+
"bin": {
|
|
10
|
+
"ailuntz-downloads": "bin/ailuntz-downloads.mjs"
|
|
11
|
+
},
|
|
12
|
+
"exports": {
|
|
13
|
+
"./client": "./src/client.mjs",
|
|
14
|
+
"./policy": "./src/channel-policy.mjs"
|
|
15
|
+
},
|
|
16
|
+
"files": [
|
|
17
|
+
"bin/",
|
|
18
|
+
"src/",
|
|
19
|
+
"README.md"
|
|
20
|
+
],
|
|
21
|
+
"dependencies": {
|
|
22
|
+
"aws4fetch": "1.0.20"
|
|
23
|
+
},
|
|
24
|
+
"license": "UNLICENSED",
|
|
25
|
+
"publishConfig": {
|
|
26
|
+
"access": "public",
|
|
27
|
+
"registry": "https://registry.npmjs.org/"
|
|
28
|
+
}
|
|
29
|
+
}
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
import fs from 'node:fs';import os from 'node:os';import path from 'node:path';import {spawn} from 'node:child_process';
|
|
2
|
+
import {validatePolicy,scalingDecision,safeToStop} from './channel-policy.mjs';
|
|
3
|
+
const runtime=process.env.AILUNTZ_RUNTIME||path.join(os.homedir(),'Library/Application Support/AiluntzDownloads');
|
|
4
|
+
const configRoot=process.env.AILUNTZ_CONFIG_DIR||runtime;
|
|
5
|
+
const definitions=JSON.parse(fs.readFileSync(configRoot+'/channels.json'));
|
|
6
|
+
if(definitions.length!==4||definitions[0].id!=='primary')throw Error('Invalid channel definitions');
|
|
7
|
+
const readyRoot=runtime+'/state/ready';fs.mkdirSync(readyRoot,{recursive:true,mode:0o700});
|
|
8
|
+
const channels=definitions.map(c=>({...c,state:c.id==='primary'?'starting':'stopped',healthy:false,requests:null,child:null,lastStart:0,drainStarted:null}));
|
|
9
|
+
const marker=c=>readyRoot+'/'+c.hostname+'.ready';
|
|
10
|
+
const remove=c=>fs.rmSync(marker(c),{force:true});
|
|
11
|
+
for(const c of channels)remove(c);
|
|
12
|
+
let policy=validatePolicy(JSON.parse(fs.readFileSync(configRoot+'/channel-policy.json'))),busySince=null,idleSince=null,stopping=false;
|
|
13
|
+
function start(c){
|
|
14
|
+
if(c.id==='primary'||c.child)return;
|
|
15
|
+
remove(c);c.state='starting';c.lastStart=Date.now();c.drainStarted=null;
|
|
16
|
+
const container=process.env.AILUNTZ_CONTAINER==='1';
|
|
17
|
+
const fd=container?null:fs.openSync(runtime+'/logs/'+c.id+'.log','a',0o600);
|
|
18
|
+
c.child=spawn(process.env.CLOUDFLARED_BIN||runtime+'/bin/cloudflared',['tunnel','--no-autoupdate','--loglevel','error','--metrics','127.0.0.1:'+c.metricsPort,'--grace-period','3m','run','--token-file',runtime+'/secrets/'+c.tokenFile],{stdio:container?['ignore','inherit','inherit']:['ignore',fd,fd]});if(fd!==null)fs.closeSync(fd);
|
|
19
|
+
c.child.once('error',()=>{remove(c);c.healthy=false;});
|
|
20
|
+
c.child.once('close',()=>{c.child=null;c.healthy=false;c.requests=null;remove(c);if(c.state==='stopping')c.state='stopped';});
|
|
21
|
+
}
|
|
22
|
+
async function observe(c){
|
|
23
|
+
if(c.state==='stopped'||c.state==='stopping')return;
|
|
24
|
+
try{
|
|
25
|
+
const [health,metrics]=await Promise.all([
|
|
26
|
+
fetch('http://127.0.0.1:'+c.metricsPort+'/ready',{signal:AbortSignal.timeout(1500)}),
|
|
27
|
+
fetch('http://127.0.0.1:'+c.metricsPort+'/metrics',{signal:AbortSignal.timeout(1500)}),
|
|
28
|
+
]);
|
|
29
|
+
const text=await metrics.text(),match=text.match(/^cloudflared_tunnel_concurrent_requests_per_tunnel(?:\{[^\n]*\})? ([\d.e+\-]+)$/m);
|
|
30
|
+
c.healthy=health.status===200;c.requests=match?Number(match[1]):null;
|
|
31
|
+
if(c.healthy&&c.state==='starting')c.state='ready';
|
|
32
|
+
}catch{c.healthy=false;c.requests=null;}
|
|
33
|
+
if(c.state==='ready'&&c.healthy){if(!fs.existsSync(marker(c)))fs.writeFileSync(marker(c),'ready\n',{mode:0o600});}
|
|
34
|
+
else remove(c);
|
|
35
|
+
}
|
|
36
|
+
async function tick(){
|
|
37
|
+
let policyError=false;
|
|
38
|
+
try{policy=validatePolicy(JSON.parse(fs.readFileSync(configRoot+'/channel-policy.json')));}catch{policyError=true;}
|
|
39
|
+
for(const c of channels){if(c.id!=='primary'&&['ready','starting','draining'].includes(c.state)&&!c.child&&Date.now()-c.lastStart>=15000){
|
|
40
|
+
// A crashed draining lane must stay withdrawn while its old links expire.
|
|
41
|
+
const oldState=c.state,oldDrain=c.drainStarted;start(c);if(oldState==='draining'){c.state=oldState;c.drainStarted=oldDrain;}
|
|
42
|
+
}}
|
|
43
|
+
await Promise.all(channels.map(observe));
|
|
44
|
+
const now=Date.now();
|
|
45
|
+
const decision=scalingDecision({policy,channels,now,busySince,idleSince});busySince=decision.busySince;idleSince=decision.idleSince;
|
|
46
|
+
if(decision.start&&!stopping)start(channels.find(c=>c.id===decision.start));
|
|
47
|
+
if(decision.resume&&!stopping){const c=channels.find(c=>c.id===decision.resume);c.state=c.healthy?'ready':'starting';c.drainStarted=null;}
|
|
48
|
+
if(decision.drain){const c=channels.find(c=>c.id===decision.drain);c.state='draining';c.drainStarted=now;remove(c);}
|
|
49
|
+
for(const c of channels){
|
|
50
|
+
if(safeToStop(c,now,policy)){c.state='stopping';remove(c);if(c.child)c.child.kill('SIGTERM');else c.state='stopped';}
|
|
51
|
+
}
|
|
52
|
+
const snapshot={checkedAt:new Date().toISOString(),policy,policyError,channels:channels.map(({id,hostname,state,healthy,requests,drainStarted})=>({id,hostname,state,healthy,requests,drainStarted}))};
|
|
53
|
+
const tmp=runtime+'/state/channels.json.tmp';fs.writeFileSync(tmp,JSON.stringify(snapshot,null,2)+'\n',{mode:0o600});fs.renameSync(tmp,runtime+'/state/channels.json');
|
|
54
|
+
}
|
|
55
|
+
async function shutdown(){
|
|
56
|
+
if(stopping)return;stopping=true;
|
|
57
|
+
// Maintenance must drain extras via policy first. On shutdown, cloudflared's
|
|
58
|
+
// own graceful exit keeps already-started requests alive as long as possible.
|
|
59
|
+
for(const c of channels){remove(c);if(c.child)c.child.kill('SIGTERM');}
|
|
60
|
+
}
|
|
61
|
+
process.on('SIGTERM',()=>void shutdown());process.on('SIGINT',()=>void shutdown());
|
|
62
|
+
while(!stopping){try{await tick();}catch{console.error('Channel check failed; keeping existing transfers.');}await new Promise(r=>setTimeout(r,5000));}
|
|
63
|
+
await Promise.all(channels.filter(c=>c.child).map(c=>new Promise(resolve=>c.child.once('close',resolve))));
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
export function validatePolicy(p, count=4) {
|
|
2
|
+
const integer=(name,min,max)=>{if(!Number.isInteger(p[name])||p[name]<min||p[name]>max)throw Error('Invalid channel policy: '+name);};
|
|
3
|
+
integer('minChannels',1,count);integer('maxChannels',p.minChannels,count);
|
|
4
|
+
integer('requestsPerChannel',1,100);integer('scaleUpSeconds',15,600);integer('idleSeconds',60,86400);
|
|
5
|
+
// Issued URLs last at most 120 seconds; allow for in-flight authorization/probes.
|
|
6
|
+
integer('drainSeconds',150,3600);
|
|
7
|
+
if(typeof p.autoScale!=='boolean')throw Error('Invalid channel policy: autoScale');
|
|
8
|
+
return p;
|
|
9
|
+
}
|
|
10
|
+
export function scalingDecision({policy,channels,now,busySince,idleSince}) {
|
|
11
|
+
const accepting=channels.filter(c=>c.state==='ready'||c.state==='starting');
|
|
12
|
+
const draining=channels.filter(c=>c.state==='draining');
|
|
13
|
+
const running=channels.filter(c=>c.state!=='stopped');
|
|
14
|
+
const ready=channels.filter(c=>c.state==='ready'&&c.healthy);
|
|
15
|
+
const known=ready.length===accepting.length&&ready.every(c=>Number.isFinite(c.requests));
|
|
16
|
+
const requests=ready.reduce((sum,c)=>sum+(c.requests||0),0);
|
|
17
|
+
let busy=known&&requests>=ready.length*policy.requestsPerChannel&&ready.length>0;
|
|
18
|
+
let idle=known&&requests<=Math.max(0,ready.length-2);
|
|
19
|
+
busySince=busy?(busySince??now):null;idleSince=idle?(idleSince??now):null;
|
|
20
|
+
const result={busySince,idleSince};
|
|
21
|
+
// A changed cap immediately removes excess lanes from new-link allocation,
|
|
22
|
+
// but they remain alive until all issued links and transfers have drained.
|
|
23
|
+
if(accepting.length>policy.maxChannels){result.drain=accepting.filter(c=>c.id!=='primary').at(-1)?.id;return result;}
|
|
24
|
+
if(accepting.length<policy.minChannels){
|
|
25
|
+
if(draining.length)result.resume=draining[0].id;
|
|
26
|
+
else if(running.length<policy.maxChannels)result.start=channels.find(c=>c.state==='stopped')?.id;
|
|
27
|
+
return result;
|
|
28
|
+
}
|
|
29
|
+
if(draining.length||accepting.some(c=>c.state==='starting'))return result;
|
|
30
|
+
if(policy.autoScale&&busySince!==null&&now-busySince>=policy.scaleUpSeconds*1000&&running.length<policy.maxChannels){
|
|
31
|
+
result.start=channels.find(c=>c.state==='stopped')?.id;result.busySince=null;result.idleSince=null;
|
|
32
|
+
}else if(accepting.length>policy.minChannels&&(!policy.autoScale||(idleSince!==null&&now-idleSince>=policy.idleSeconds*1000))){
|
|
33
|
+
result.drain=[...ready].filter(c=>c.id!=='primary').sort((a,b)=>a.requests-b.requests)[0]?.id;result.idleSince=null;result.busySince=null;
|
|
34
|
+
}
|
|
35
|
+
return result;
|
|
36
|
+
}
|
|
37
|
+
export function safeToStop(channel, now, policy) {
|
|
38
|
+
return channel.state==='draining'&&Number.isFinite(channel.requests)&&channel.requests===0&&now-channel.drainStarted>=policy.drainSeconds*1000;
|
|
39
|
+
}
|
package/src/client.mjs
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
// This module uses Web APIs only, so the same protocol works in Node and Deno.
|
|
2
|
+
const encoder = new TextEncoder();
|
|
3
|
+
const hex = bytes => Array.from(new Uint8Array(bytes), x => x.toString(16).padStart(2, '0')).join('');
|
|
4
|
+
export async function signedRequest({clientId, secret, path, body, now = Date.now(), nonce = crypto.randomUUID()}) {
|
|
5
|
+
const text = JSON.stringify(body), timestamp = String(Math.floor(now / 1000));
|
|
6
|
+
const digest = hex(await crypto.subtle.digest('SHA-256', encoder.encode(text)));
|
|
7
|
+
const message = ['download-agent-v1', clientId, 'POST', path, timestamp, nonce, digest].join('\n');
|
|
8
|
+
const key = await crypto.subtle.importKey('raw', encoder.encode(secret), {name: 'HMAC', hash: 'SHA-256'}, false, ['sign']);
|
|
9
|
+
const signature = hex(await crypto.subtle.sign('HMAC', key, encoder.encode(message)));
|
|
10
|
+
return {method: 'POST', body: text, redirect: 'error', headers: {
|
|
11
|
+
'Content-Type': 'application/json',
|
|
12
|
+
'X-Download-Client': clientId, 'X-Download-Timestamp': timestamp,
|
|
13
|
+
'X-Download-Nonce': nonce, 'X-Download-Signature': signature,
|
|
14
|
+
}};
|
|
15
|
+
}
|
package/src/server.mjs
ADDED
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import http from 'node:http';
|
|
2
|
+
import fs from 'node:fs';
|
|
3
|
+
import {createDownloadService} from './service.mjs';
|
|
4
|
+
|
|
5
|
+
export async function serve(runtime) {
|
|
6
|
+
const config = JSON.parse(fs.readFileSync(runtime + '/secrets/agent.json'));
|
|
7
|
+
const channels = JSON.parse(fs.readFileSync((process.env.AILUNTZ_CONFIG_DIR || runtime) + '/channels.json'));
|
|
8
|
+
const service = createDownloadService({config, channels, snapshot: () => {
|
|
9
|
+
const state = JSON.parse(fs.readFileSync(runtime + '/state/channels.json'));
|
|
10
|
+
// Markers disappear at the start of draining, before the next snapshot.
|
|
11
|
+
state.channels = state.channels.filter(c => fs.existsSync(runtime + '/state/ready/' + c.hostname + '.ready'));
|
|
12
|
+
return state;
|
|
13
|
+
}});
|
|
14
|
+
const server = http.createServer({requestTimeout: 10000, headersTimeout: 5000, maxHeaderSize: 8192}, async (req, res) => {
|
|
15
|
+
try {
|
|
16
|
+
const chunks = []; let size = 0;
|
|
17
|
+
for await (const chunk of req) {
|
|
18
|
+
size += chunk.length;
|
|
19
|
+
if (size > 8192) {
|
|
20
|
+
res.writeHead(413, {'Connection': 'close', 'Cache-Control': 'no-store'}); res.end(); return;
|
|
21
|
+
}
|
|
22
|
+
chunks.push(chunk);
|
|
23
|
+
}
|
|
24
|
+
const method = req.method || 'GET', body = Buffer.concat(chunks);
|
|
25
|
+
const input = new Request('http://127.0.0.1:19081' + req.url, {
|
|
26
|
+
method, headers: req.headers,
|
|
27
|
+
...(!['GET', 'HEAD'].includes(method) ? {body} : {}),
|
|
28
|
+
});
|
|
29
|
+
const result = await service.handle(input);
|
|
30
|
+
res.writeHead(result.status, Object.fromEntries(result.headers));
|
|
31
|
+
res.end(Buffer.from(await result.arrayBuffer()));
|
|
32
|
+
} catch {
|
|
33
|
+
if (!res.headersSent) res.writeHead(400, {'Cache-Control': 'no-store'});
|
|
34
|
+
res.end();
|
|
35
|
+
}
|
|
36
|
+
});
|
|
37
|
+
server.maxHeadersCount = 40;
|
|
38
|
+
server.maxConnections = 64;
|
|
39
|
+
await new Promise((resolve, reject) => { server.once('error', reject); server.listen(19081, '127.0.0.1', resolve); });
|
|
40
|
+
const writeStatus = () => {
|
|
41
|
+
const data = {version: '1.1.0', pid: process.pid, checkedAt: new Date().toISOString(), ...service.status()};
|
|
42
|
+
const tmp = runtime + '/state/agent.json.tmp';
|
|
43
|
+
fs.writeFileSync(tmp, JSON.stringify(data, null, 2) + '\n', {mode: 0o600});
|
|
44
|
+
fs.renameSync(tmp, runtime + '/state/agent.json');
|
|
45
|
+
};
|
|
46
|
+
writeStatus();
|
|
47
|
+
const timer = setInterval(writeStatus, 5000);
|
|
48
|
+
const stop = () => {
|
|
49
|
+
clearInterval(timer); server.close(() => process.exit(0));
|
|
50
|
+
setTimeout(() => { server.closeAllConnections(); process.exit(0); }, 12000).unref();
|
|
51
|
+
};
|
|
52
|
+
process.once('SIGTERM', stop); process.once('SIGINT', stop);
|
|
53
|
+
console.log('Download agent listening on loopback.');
|
|
54
|
+
}
|
package/src/service.mjs
ADDED
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
import {createHmac, createHash, timingSafeEqual} from 'node:crypto';
|
|
2
|
+
import {AwsClient} from 'aws4fetch';
|
|
3
|
+
|
|
4
|
+
export class Problem extends Error {
|
|
5
|
+
constructor(status, code) { super(code); this.status = status; }
|
|
6
|
+
}
|
|
7
|
+
const fail = (status, code) => { throw new Problem(status, code); };
|
|
8
|
+
const response = (body, status = 200) => Response.json(body, {status, headers: {
|
|
9
|
+
'Cache-Control': 'private, no-store, max-age=0', 'X-Content-Type-Options': 'nosniff',
|
|
10
|
+
}});
|
|
11
|
+
export function validateConfig(config) {
|
|
12
|
+
if (!config?.s3?.accessKeyId || !config.s3.secretAccessKey || !config.s3.region) throw Error('Missing storage configuration');
|
|
13
|
+
if (!config.clients || !Object.keys(config.clients).length) throw Error('Missing client configuration');
|
|
14
|
+
for (const [id, client] of Object.entries(config.clients)) {
|
|
15
|
+
if (!/^[a-z0-9-]{1,50}$/.test(id) || !/^[A-Za-z0-9_-]{43,128}$/.test(client.secret) ||
|
|
16
|
+
!Array.isArray(client.buckets) || !client.buckets.length ||
|
|
17
|
+
client.buckets.some(b => !/^[a-z0-9][a-z0-9-]{1,61}[a-z0-9]$/.test(b))) throw Error('Invalid client configuration');
|
|
18
|
+
}
|
|
19
|
+
return config;
|
|
20
|
+
}
|
|
21
|
+
export function createDownloadService({config, channels, snapshot, request = fetch, now = Date.now}) {
|
|
22
|
+
validateConfig(config);
|
|
23
|
+
if (!Array.isArray(channels) || !channels.length || channels.length > 4 ||
|
|
24
|
+
channels.some(c => !/^[a-z0-9.-]+$/.test(c.hostname))) throw Error('Invalid channels');
|
|
25
|
+
const aws = new AwsClient({...config.s3, service: 's3'}), seen = new Map();
|
|
26
|
+
let cursor = 0, active = 0;
|
|
27
|
+
const stats = {issued: 0, metadata: 0, denied: 0, unavailable: 0};
|
|
28
|
+
const accepting = () => {
|
|
29
|
+
const state = snapshot();
|
|
30
|
+
if (now() - Date.parse(state.checkedAt) > 15000 || Date.parse(state.checkedAt) > now() + 5000 ||
|
|
31
|
+
!Number.isFinite(Date.parse(state.checkedAt))) fail(503, 'storage_unavailable');
|
|
32
|
+
return channels.filter(c => state.channels.some(s => s.id === c.id && s.hostname === c.hostname && s.state === 'ready' && s.healthy));
|
|
33
|
+
};
|
|
34
|
+
async function sign(endpoint, bucket, key, ttl, method, filename) {
|
|
35
|
+
const url = new URL(endpoint + '/' + [bucket, ...key.split('/')].map(encodeURIComponent).join('/'));
|
|
36
|
+
url.searchParams.set('X-Amz-Expires', String(ttl));
|
|
37
|
+
if (filename) url.searchParams.set('response-content-disposition', `attachment; filename*=UTF-8''${encodeURIComponent(filename)}`);
|
|
38
|
+
return (await aws.sign(url.toString(), {method, aws: {signQuery: true}})).url;
|
|
39
|
+
}
|
|
40
|
+
async function select(bucket, key) {
|
|
41
|
+
const candidates = accepting();
|
|
42
|
+
if (!candidates.length) fail(503, 'storage_unavailable');
|
|
43
|
+
const order = candidates.map((_, i) => candidates[(cursor + i) % candidates.length]);
|
|
44
|
+
cursor = (cursor + 1) % candidates.length;
|
|
45
|
+
async function check(channel) {
|
|
46
|
+
const endpoint = 'https://' + channel.hostname;
|
|
47
|
+
const ready = await request(endpoint + '/channel-ready', {method: 'HEAD', redirect: 'error', signal: AbortSignal.timeout(2000)});
|
|
48
|
+
if (ready.status !== 200) throw Error('Unavailable');
|
|
49
|
+
const metadata = await request(await sign(endpoint, bucket, key, 15, 'HEAD'), {method: 'HEAD', redirect: 'error', signal: AbortSignal.timeout(3000)});
|
|
50
|
+
if (metadata.status !== 200 || !accepting().some(c => c.id === channel.id)) throw Error('Unavailable');
|
|
51
|
+
return {endpoint, metadata};
|
|
52
|
+
}
|
|
53
|
+
try { return await check(order[0]); }
|
|
54
|
+
catch {
|
|
55
|
+
try { return await Promise.any(order.slice(1).map(check)); }
|
|
56
|
+
catch { fail(503, 'storage_unavailable'); }
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
function authenticate(req, text, pathname) {
|
|
60
|
+
const id = req.headers.get('x-download-client') || '', client = config.clients[id];
|
|
61
|
+
const timestamp = req.headers.get('x-download-timestamp') || '', nonce = req.headers.get('x-download-nonce') || '';
|
|
62
|
+
const signature = req.headers.get('x-download-signature') || '';
|
|
63
|
+
if (!Object.hasOwn(config.clients, id) || !client || !/^\d{10}$/.test(timestamp) ||
|
|
64
|
+
!/^[A-Za-z0-9_-]{20,100}$/.test(nonce) || !/^[0-9a-f]{64}$/.test(signature)) fail(401, 'unauthorized');
|
|
65
|
+
const age = Math.floor(now() / 1000) - Number(timestamp);
|
|
66
|
+
if (age < -5 || age > 30) fail(401, 'unauthorized');
|
|
67
|
+
const digest = createHash('sha256').update(text).digest('hex');
|
|
68
|
+
const message = ['download-agent-v1', id, req.method, pathname, timestamp, nonce, digest].join('\n');
|
|
69
|
+
const expected = createHmac('sha256', client.secret).update(message).digest();
|
|
70
|
+
if (!timingSafeEqual(expected, Buffer.from(signature, 'hex'))) fail(401, 'unauthorized');
|
|
71
|
+
for (const [k, expires] of seen) if (expires < now()) seen.delete(k);
|
|
72
|
+
const nonceKey = id + ':' + nonce;
|
|
73
|
+
if (seen.has(nonceKey)) fail(409, 'replayed_request');
|
|
74
|
+
if (seen.size >= 10000) fail(429, 'busy');
|
|
75
|
+
// Consume before I/O: simultaneous copies cannot both issue links.
|
|
76
|
+
seen.set(nonceKey, (Number(timestamp) + 31) * 1000);
|
|
77
|
+
return client;
|
|
78
|
+
}
|
|
79
|
+
async function handle(req) {
|
|
80
|
+
let reserved = false;
|
|
81
|
+
try {
|
|
82
|
+
const url = new URL(req.url);
|
|
83
|
+
if (req.method === 'GET' && url.pathname === '/healthz' && !url.search) return response({ok: true});
|
|
84
|
+
if (req.method !== 'POST' || url.search || !['/v1/downloads', '/v1/objects/stat'].includes(url.pathname)) fail(404, 'not_found');
|
|
85
|
+
if (req.headers.has('origin')) fail(403, 'server_only');
|
|
86
|
+
if (!req.headers.get('content-type')?.startsWith('application/json')) fail(415, 'json_required');
|
|
87
|
+
// Network adapter enforces this limit while reading, before buffering.
|
|
88
|
+
const text = await req.text();
|
|
89
|
+
if (Buffer.byteLength(text) > 8192) fail(413, 'request_too_large');
|
|
90
|
+
const client = authenticate(req, text, url.pathname);
|
|
91
|
+
let body; try { body = JSON.parse(text); } catch { fail(400, 'invalid_input'); }
|
|
92
|
+
if (!body || typeof body !== 'object' || Array.isArray(body)) fail(400, 'invalid_input');
|
|
93
|
+
const {bucket, key, filename, expiresAt} = body;
|
|
94
|
+
if (!client.buckets.includes(bucket)) fail(403, 'bucket_not_allowed');
|
|
95
|
+
if (typeof key !== 'string' || Buffer.byteLength(key) > 1024 || /[\\\x00-\x1f\x7f]/.test(key) ||
|
|
96
|
+
key.split('/').some(s => !s || s === '.' || s === '..')) fail(400, 'invalid_input');
|
|
97
|
+
const download = url.pathname === '/v1/downloads';
|
|
98
|
+
if (download && (typeof filename !== 'string' || !filename.length || filename.length > 200 ||
|
|
99
|
+
/[\/\\\x00-\x1f\x7f]/.test(filename) || !Number.isSafeInteger(expiresAt) || expiresAt > now() + 120000)) fail(400, 'invalid_input');
|
|
100
|
+
if (download && expiresAt <= now()) fail(403, 'grant_expired');
|
|
101
|
+
if (active >= 32) fail(429, 'busy');
|
|
102
|
+
active++; reserved = true;
|
|
103
|
+
const {endpoint, metadata} = await select(bucket, key);
|
|
104
|
+
if (!download) {
|
|
105
|
+
const size = Number(metadata.headers.get('content-length'));
|
|
106
|
+
if (!Number.isSafeInteger(size) || size < 1) fail(503, 'storage_unavailable');
|
|
107
|
+
stats.metadata++; return response({size_bytes: size});
|
|
108
|
+
}
|
|
109
|
+
const ttl = Math.min(120, Math.floor((expiresAt - now()) / 1000));
|
|
110
|
+
if (ttl < 1) fail(403, 'grant_expired');
|
|
111
|
+
const signed = await sign(endpoint, bucket, key, ttl, 'GET', filename);
|
|
112
|
+
if (expiresAt <= now()) fail(403, 'grant_expired');
|
|
113
|
+
stats.issued++;
|
|
114
|
+
return response({url: signed, filename, expires_in: ttl});
|
|
115
|
+
} catch (error) {
|
|
116
|
+
const status = error instanceof Problem ? error.status : 503;
|
|
117
|
+
stats[status >= 500 ? 'unavailable' : 'denied']++;
|
|
118
|
+
return response({error: error instanceof Problem ? error.message : 'storage_unavailable'}, status);
|
|
119
|
+
} finally { if (reserved) active--; }
|
|
120
|
+
}
|
|
121
|
+
return {handle, status: () => ({...stats, active})};
|
|
122
|
+
}
|