@shhhum/xftp-web 0.4.0 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. package/README.md +15 -118
  2. package/dist/agent.d.ts +9 -29
  3. package/dist/agent.js +98 -238
  4. package/dist/agent.js.map +1 -1
  5. package/dist/client.d.ts +16 -17
  6. package/dist/client.js +21 -27
  7. package/dist/client.js.map +1 -1
  8. package/dist/crypto/digest.d.ts +0 -5
  9. package/dist/crypto/digest.js +0 -10
  10. package/dist/crypto/digest.js.map +1 -1
  11. package/dist/crypto/file.d.ts +0 -12
  12. package/dist/crypto/file.js +0 -48
  13. package/dist/crypto/file.js.map +1 -1
  14. package/dist/crypto/identity.d.ts +0 -1
  15. package/dist/crypto/keys.d.ts +0 -1
  16. package/dist/crypto/keys.js +0 -1
  17. package/dist/crypto/keys.js.map +1 -1
  18. package/dist/crypto/padding.d.ts +0 -1
  19. package/dist/crypto/secretbox.d.ts +0 -1
  20. package/dist/download.d.ts +0 -1
  21. package/dist/protocol/address.d.ts +0 -1
  22. package/dist/protocol/chunks.d.ts +0 -1
  23. package/dist/protocol/client.d.ts +0 -1
  24. package/dist/protocol/commands.d.ts +1 -10
  25. package/dist/protocol/commands.js +1 -15
  26. package/dist/protocol/commands.js.map +1 -1
  27. package/dist/protocol/description.d.ts +0 -1
  28. package/dist/protocol/encoding.d.ts +0 -1
  29. package/dist/protocol/handshake.d.ts +0 -1
  30. package/dist/protocol/transmission.d.ts +0 -1
  31. package/dist-web/assets/__vite-browser-external-9wXp6ZBx.js +1 -0
  32. package/dist-web/assets/__vite-browser-external.js +1 -0
  33. package/dist-web/assets/index.css +1 -0
  34. package/dist-web/assets/index.js +1468 -0
  35. package/dist-web/crypto.worker.js +1413 -0
  36. package/dist-web/index.html +15 -0
  37. package/package.json +4 -5
  38. package/src/agent.ts +108 -310
  39. package/src/client.ts +38 -40
  40. package/src/crypto/digest.ts +2 -15
  41. package/src/crypto/file.ts +0 -83
  42. package/src/crypto/keys.ts +0 -1
  43. package/src/protocol/commands.ts +2 -22
  44. package/web/crypto-backend.ts +122 -0
  45. package/web/crypto.worker.ts +313 -0
  46. package/web/download.ts +140 -0
  47. package/web/index.html +15 -0
  48. package/web/main.ts +30 -0
  49. package/web/progress.ts +52 -0
  50. package/web/servers.json +18 -0
  51. package/web/servers.ts +16 -0
  52. package/web/style.css +103 -0
  53. package/web/upload.ts +171 -0
  54. package/src/index.ts +0 -4
package/README.md CHANGED
@@ -2,116 +2,13 @@
2
2
 
3
3
  Browser-compatible XFTP file transfer client in TypeScript.
4
4
 
5
- ## Installation
6
-
7
- ```bash
8
- npm install xftp-web
9
- ```
10
-
11
- ## Usage
12
-
13
- ```typescript
14
- import {
15
- XFTPAgent,
16
- parseXFTPServer,
17
- sendFile, receiveFile, deleteFile,
18
- XFTPRetriableError, XFTPPermanentError, isRetriable,
19
- } from "xftp-web"
20
-
21
- // Create agent (manages connections)
22
- const agent = new XFTPAgent()
23
-
24
- const servers = [
25
- parseXFTPServer("xftp://server1..."),
26
- parseXFTPServer("xftp://server2..."),
27
- parseXFTPServer("xftp://server3..."),
28
- ]
29
-
30
- // Upload (from Uint8Array)
31
- const {rcvDescriptions, sndDescription, uri} = await sendFile(
32
- agent, servers, fileBytes, "photo.jpg",
33
- {onProgress: (uploaded, total) => console.log(`${uploaded}/${total}`)}
34
- )
35
-
36
- // Upload (streaming — constant memory, no full-file buffer)
37
- const file = inputEl.files[0]
38
- const result = await sendFile(
39
- agent, servers, file.stream(), file.size, file.name,
40
- {onProgress: (uploaded, total) => console.log(`${uploaded}/${total}`)}
41
- )
42
-
43
- // Download
44
- const {header, content} = await receiveFile(agent, uri, {
45
- onProgress: (downloaded, total) => console.log(`${downloaded}/${total}`)
46
- })
47
-
48
- // Delete (requires sender description from upload)
49
- await deleteFile(agent, sndDescription)
50
-
51
- // Cleanup
52
- agent.close()
53
- ```
54
-
55
- ### Advanced usage
56
-
57
- For streaming encryption (avoids buffering the full encrypted file) or worker-based uploads:
58
-
59
- ```typescript
60
- import {
61
- encryptFileForUpload, uploadFile, downloadFile,
62
- decodeDescriptionURI,
63
- } from "xftp-web"
64
-
65
- // Streaming encryption — encrypted slices emitted via callback
66
- const metadata = await encryptFileForUpload(fileBytes, "photo.jpg", {
67
- onSlice: (data) => { /* write to OPFS, IndexedDB, etc. */ },
68
- onProgress: (done, total) => {},
69
- })
70
- // metadata has {digest, key, nonce, chunkSizes} but no encData
71
-
72
- // Upload with custom chunk reader (e.g. reading from OPFS)
73
- const result = await uploadFile(agent, servers, metadata, {
74
- readChunk: (offset, size) => readFromStorage(offset, size),
75
- })
76
-
77
- // Download with FileDescription object
78
- const fd = decodeDescriptionURI(uri)
79
- const {header, content} = await downloadFile(agent, fd)
80
- ```
81
-
82
- ### Upload options
83
-
84
- ```typescript
85
- await sendFile(agent, servers, fileBytes, "photo.jpg", {
86
- onProgress: (uploaded, total) => {}, // progress callback
87
- auth: basicAuthBytes, // BasicAuth for auth-required servers
88
- numRecipients: 3, // multiple independent download credentials (default: 1)
89
- })
90
- ```
91
-
92
- ### Error handling
93
-
94
- ```typescript
95
- try {
96
- await sendFile(agent, servers, fileBytes, "photo.jpg")
97
- } catch (e) {
98
- if (e instanceof XFTPRetriableError) {
99
- // Network/timeout/session errors — safe to retry
100
- } else if (e instanceof XFTPPermanentError) {
101
- // AUTH, NO_FILE, BLOCKED, etc. — do not retry
102
- }
103
- // or use: isRetriable(e)
104
- }
105
- ```
106
-
107
- ## Development
108
-
109
- ### Prerequisites
5
+ ## Prerequisites
110
6
 
111
7
  - Haskell toolchain with `cabal` (to build `xftp-server`)
112
8
  - Node.js 20+
9
+ - Chromium system dependencies (see below)
113
10
 
114
- ### Setup
11
+ ## Setup
115
12
 
116
13
  ```bash
117
14
  # Build the XFTP server binary (from repo root)
@@ -120,31 +17,31 @@ cabal build xftp-server
120
17
  # Install JS dependencies
121
18
  cd xftp-web
122
19
  npm install
20
+
21
+ # Install Chromium for Playwright (browser tests)
22
+ npx playwright install chromium
123
23
  ```
124
24
 
125
- ### Running tests
25
+ If Chromium fails to launch due to missing system libraries, install them with:
126
26
 
127
27
  ```bash
128
- npm run test
28
+ # Requires root
29
+ npx playwright install-deps chromium
129
30
  ```
130
31
 
131
- The `pretest` script automatically installs Chromium and sets up the libsodium symlink. The browser test starts an `xftp-server` instance on port 7000 via `globalSetup`.
132
-
133
- If Chromium fails to launch due to missing system libraries:
32
+ ## Running tests
134
33
 
135
34
  ```bash
136
- # Requires root
137
- npx playwright install-deps chromium
35
+ # Browser round-trip test (vitest + Playwright headless Chromium)
36
+ npm run test
138
37
  ```
139
38
 
140
- ### Build
39
+ The browser test automatically starts an `xftp-server` instance on port 7000 via `globalSetup`, using certs from `tests/fixtures/`.
40
+
41
+ ## Build
141
42
 
142
43
  ```bash
143
44
  npm run build
144
45
  ```
145
46
 
146
47
  Output goes to `dist/`.
147
-
148
- ## License
149
-
150
- [AGPL-3.0-only](https://www.gnu.org/licenses/agpl-3.0.html)
package/dist/agent.d.ts CHANGED
@@ -1,7 +1,6 @@
1
- export { prepareEncryption, type EncryptionParams } from "./crypto/file.js";
2
1
  import { type FileDescription } from "./protocol/description.js";
3
- import { XFTPAgent } from "./client.js";
4
- export { XFTPAgent, type TransportConfig, XFTPRetriableError, XFTPPermanentError, isRetriable, categorizeError, humanReadableMessage } from "./client.js";
2
+ import { type XFTPClientAgent } from "./client.js";
3
+ export { newXFTPAgent, closeXFTPAgent, type XFTPClientAgent, type TransportConfig } from "./client.js";
5
4
  import type { XFTPServer } from "./protocol/address.js";
6
5
  import type { FileHeader } from "./crypto/file.js";
7
6
  export interface EncryptedFileMetadata {
@@ -14,7 +13,7 @@ export interface EncryptedFileInfo extends EncryptedFileMetadata {
14
13
  encData: Uint8Array;
15
14
  }
16
15
  export interface UploadResult {
17
- rcvDescriptions: FileDescription[];
16
+ rcvDescription: FileDescription;
18
17
  sndDescription: FileDescription;
19
18
  uri: string;
20
19
  }
@@ -24,29 +23,13 @@ export interface DownloadResult {
24
23
  }
25
24
  export declare function encodeDescriptionURI(fd: FileDescription): string;
26
25
  export declare function decodeDescriptionURI(fragment: string): FileDescription;
27
- export interface EncryptForUploadOptions {
28
- onProgress?: (done: number, total: number) => void;
29
- onSlice?: (data: Uint8Array) => void | Promise<void>;
30
- }
31
- export declare function encryptFileForUpload(source: Uint8Array, fileName: string, options: EncryptForUploadOptions & {
32
- onSlice: NonNullable<EncryptForUploadOptions['onSlice']>;
33
- }): Promise<EncryptedFileMetadata>;
34
- export declare function encryptFileForUpload(source: Uint8Array, fileName: string, options?: EncryptForUploadOptions): Promise<EncryptedFileInfo>;
26
+ export declare function encryptFileForUpload(source: Uint8Array, fileName: string): EncryptedFileInfo;
35
27
  export interface UploadOptions {
36
28
  onProgress?: (uploaded: number, total: number) => void;
37
29
  redirectThreshold?: number;
38
30
  readChunk?: (offset: number, size: number) => Promise<Uint8Array>;
39
- auth?: Uint8Array;
40
- numRecipients?: number;
41
- }
42
- export declare function uploadFile(agent: XFTPAgent, servers: XFTPServer[], encrypted: EncryptedFileMetadata, options?: UploadOptions): Promise<UploadResult>;
43
- export interface SendFileOptions {
44
- onProgress?: (uploaded: number, total: number) => void;
45
- auth?: Uint8Array;
46
- numRecipients?: number;
47
31
  }
48
- export declare function sendFile(agent: XFTPAgent, servers: XFTPServer[], source: Uint8Array, fileName: string, options?: SendFileOptions): Promise<UploadResult>;
49
- export declare function sendFile(agent: XFTPAgent, servers: XFTPServer[], source: AsyncIterable<Uint8Array>, sourceSize: number, fileName: string, options?: SendFileOptions): Promise<UploadResult>;
32
+ export declare function uploadFile(agent: XFTPClientAgent, server: XFTPServer, encrypted: EncryptedFileMetadata, options?: UploadOptions): Promise<UploadResult>;
50
33
  export interface RawDownloadedChunk {
51
34
  chunkNo: number;
52
35
  dhSecret: Uint8Array;
@@ -56,11 +39,8 @@ export interface RawDownloadedChunk {
56
39
  }
57
40
  export interface DownloadRawOptions {
58
41
  onProgress?: (downloaded: number, total: number) => void;
42
+ concurrency?: number;
59
43
  }
60
- export declare function downloadFileRaw(agent: XFTPAgent, fd: FileDescription, onRawChunk: (chunk: RawDownloadedChunk) => Promise<void>, options?: DownloadRawOptions): Promise<FileDescription>;
61
- export declare function downloadFile(agent: XFTPAgent, fd: FileDescription, onProgress?: (downloaded: number, total: number) => void): Promise<DownloadResult>;
62
- export declare function receiveFile(agent: XFTPAgent, uri: string, options?: {
63
- onProgress?: (downloaded: number, total: number) => void;
64
- }): Promise<DownloadResult>;
65
- export declare function deleteFile(agent: XFTPAgent, sndDescription: FileDescription): Promise<void>;
66
- //# sourceMappingURL=agent.d.ts.map
44
+ export declare function downloadFileRaw(agent: XFTPClientAgent, fd: FileDescription, onRawChunk: (chunk: RawDownloadedChunk) => Promise<void>, options?: DownloadRawOptions): Promise<FileDescription>;
45
+ export declare function downloadFile(agent: XFTPClientAgent, fd: FileDescription, onProgress?: (downloaded: number, total: number) => void): Promise<DownloadResult>;
46
+ export declare function deleteFile(agent: XFTPClientAgent, sndDescription: FileDescription): Promise<void>;
package/dist/agent.js CHANGED
@@ -3,16 +3,13 @@
3
3
  // Combines all building blocks: encryption, chunking, XFTP client commands,
4
4
  // file descriptions, and DEFLATE-compressed URI encoding.
5
5
  import pako from "pako";
6
- import { encryptFileAsync, prepareEncryption } from "./crypto/file.js";
7
- import { sbInit, sbEncryptChunk, sbAuth } from "./crypto/secretbox.js";
8
- import { concatBytes, encodeInt64 } from "./protocol/encoding.js";
9
- export { prepareEncryption } from "./crypto/file.js";
6
+ import { encryptFile, encodeFileHeader } from "./crypto/file.js";
10
7
  import { generateEd25519KeyPair, encodePubKeyEd25519, encodePrivKeyEd25519, decodePrivKeyEd25519, ed25519KeyPairFromSeed } from "./crypto/keys.js";
11
- import { sha512Streaming, sha512Init, sha512Update, sha512Final } from "./crypto/digest.js";
12
- import { prepareChunkSpecs, getChunkDigest } from "./protocol/chunks.js";
8
+ import { sha512Streaming } from "./crypto/digest.js";
9
+ import { prepareChunkSizes, prepareChunkSpecs, getChunkDigest, fileSizeLen, authTagSize } from "./protocol/chunks.js";
13
10
  import { encodeFileDescription, decodeFileDescription, validateFileDescription, base64urlEncode, base64urlDecode } from "./protocol/description.js";
14
- import { createXFTPChunk, addXFTPRecipients, uploadXFTPChunk, downloadXFTPChunk, downloadXFTPChunkRaw, deleteXFTPChunk, ackXFTPChunk } from "./client.js";
15
- export { XFTPAgent, XFTPRetriableError, XFTPPermanentError, isRetriable, categorizeError, humanReadableMessage } from "./client.js";
11
+ import { createXFTPChunk, uploadXFTPChunk, downloadXFTPChunk, downloadXFTPChunkRaw, deleteXFTPChunk } from "./client.js";
12
+ export { newXFTPAgent, closeXFTPAgent } from "./client.js";
16
13
  import { processDownloadedFile, decryptReceivedChunk } from "./download.js";
17
14
  import { formatXFTPServer, parseXFTPServer } from "./protocol/address.js";
18
15
  // -- URI encoding/decoding (RFC section 4.1: DEFLATE + base64url)
@@ -30,61 +27,25 @@ export function decodeDescriptionURI(fragment) {
30
27
  throw new Error("decodeDescriptionURI: " + err);
31
28
  return fd;
32
29
  }
33
- export async function encryptFileForUpload(source, fileName, options) {
34
- const { onProgress, onSlice } = options ?? {};
35
- const { fileHdr, key, nonce, fileSize, encSize, chunkSizes } = prepareEncryption(source.length, fileName);
36
- if (onSlice) {
37
- const hashState = sha512Init();
38
- await encryptFileAsync(source, fileHdr, key, nonce, fileSize, encSize, onProgress, (data) => {
39
- sha512Update(hashState, data);
40
- return onSlice(data);
41
- });
42
- const digest = sha512Final(hashState);
43
- return { digest, key, nonce, chunkSizes };
44
- }
45
- else {
46
- const encData = await encryptFileAsync(source, fileHdr, key, nonce, fileSize, encSize, onProgress);
47
- const digest = sha512Streaming([encData]);
48
- console.log(`[AGENT-DBG] encrypt: encData.len=${encData.length} digest=${_dbgHex(digest, 64)} chunkSizes=[${chunkSizes.join(',')}]`);
49
- return { encData, digest, key, nonce, chunkSizes };
50
- }
30
+ // -- Upload
31
+ export function encryptFileForUpload(source, fileName) {
32
+ const key = new Uint8Array(32);
33
+ const nonce = new Uint8Array(24);
34
+ crypto.getRandomValues(key);
35
+ crypto.getRandomValues(nonce);
36
+ const fileHdr = encodeFileHeader({ fileName, fileExtra: null });
37
+ const fileSize = BigInt(fileHdr.length + source.length);
38
+ const payloadSize = Number(fileSize) + fileSizeLen + authTagSize;
39
+ const chunkSizes = prepareChunkSizes(payloadSize);
40
+ const encSize = BigInt(chunkSizes.reduce((a, b) => a + b, 0));
41
+ const encData = encryptFile(source, fileHdr, key, nonce, fileSize, encSize);
42
+ const digest = sha512Streaming([encData]);
43
+ console.log(`[AGENT-DBG] encrypt: encData.len=${encData.length} digest=${_dbgHex(digest, 64)} chunkSizes=[${chunkSizes.join(',')}]`);
44
+ return { encData, digest, key, nonce, chunkSizes };
51
45
  }
52
46
  const DEFAULT_REDIRECT_THRESHOLD = 400;
53
- const MAX_RECIPIENTS_PER_REQUEST = 200; // each key is ~46 bytes; 200 keys fit within 16KB XFTP block
54
- async function uploadSingleChunk(agent, server, chunkNo, chunkData, chunkSize, numRecipients, auth) {
55
- const sndKp = generateEd25519KeyPair();
56
- const rcvKps = Array.from({ length: numRecipients }, () => generateEd25519KeyPair());
57
- const chunkDigest = getChunkDigest(chunkData);
58
- const fileInfo = {
59
- sndKey: encodePubKeyEd25519(sndKp.publicKey),
60
- size: chunkSize,
61
- digest: chunkDigest
62
- };
63
- const firstBatch = Math.min(numRecipients, MAX_RECIPIENTS_PER_REQUEST);
64
- const firstBatchKeys = rcvKps.slice(0, firstBatch).map(kp => encodePubKeyEd25519(kp.publicKey));
65
- const { senderId, recipientIds: firstIds } = await createXFTPChunk(agent, server, sndKp.privateKey, fileInfo, firstBatchKeys, auth);
66
- const allRecipientIds = [...firstIds];
67
- let added = firstBatch;
68
- while (added < numRecipients) {
69
- const batchSize = Math.min(numRecipients - added, MAX_RECIPIENTS_PER_REQUEST);
70
- const batchKeys = rcvKps.slice(added, added + batchSize).map(kp => encodePubKeyEd25519(kp.publicKey));
71
- const moreIds = await addXFTPRecipients(agent, server, sndKp.privateKey, senderId, batchKeys);
72
- allRecipientIds.push(...moreIds);
73
- added += batchSize;
74
- }
75
- await uploadXFTPChunk(agent, server, sndKp.privateKey, senderId, chunkData);
76
- return {
77
- chunkNo, senderId, senderKey: sndKp.privateKey,
78
- recipients: allRecipientIds.map((rid, ri) => ({
79
- recipientId: rid, recipientKey: rcvKps[ri].privateKey
80
- })),
81
- chunkSize, digest: chunkDigest, server
82
- };
83
- }
84
- export async function uploadFile(agent, servers, encrypted, options) {
85
- if (servers.length === 0)
86
- throw new Error("uploadFile: servers list is empty");
87
- const { onProgress, redirectThreshold, readChunk: readChunkOpt, auth, numRecipients = 1 } = options ?? {};
47
+ export async function uploadFile(agent, server, encrypted, options) {
48
+ const { onProgress, redirectThreshold, readChunk: readChunkOpt } = options ?? {};
88
49
  const readChunk = readChunkOpt
89
50
  ? readChunkOpt
90
51
  : ('encData' in encrypted
@@ -92,123 +53,44 @@ export async function uploadFile(agent, servers, encrypted, options) {
92
53
  : () => { throw new Error("uploadFile: readChunk required when encData is absent"); });
93
54
  const total = encrypted.chunkSizes.reduce((a, b) => a + b, 0);
94
55
  const specs = prepareChunkSpecs(encrypted.chunkSizes);
95
- // Pre-assign servers and group by server (matching Haskell groupAllOn)
96
- const chunkJobs = specs.map((spec, i) => ({
97
- index: i,
98
- spec,
99
- server: servers[Math.floor(Math.random() * servers.length)]
100
- }));
101
- const byServer = new Map();
102
- for (const job of chunkJobs) {
103
- const key = formatXFTPServer(job.server);
104
- if (!byServer.has(key))
105
- byServer.set(key, []);
106
- byServer.get(key).push(job);
107
- }
108
- // Upload groups in parallel, sequential within each group
109
- const sentChunks = new Array(specs.length);
56
+ const sentChunks = [];
110
57
  let uploaded = 0;
111
- await Promise.all([...byServer.values()].map(async (jobs) => {
112
- for (const { index, spec, server } of jobs) {
113
- const chunkNo = index + 1;
114
- const chunkData = await readChunk(spec.chunkOffset, spec.chunkSize);
115
- sentChunks[index] = await uploadSingleChunk(agent, server, chunkNo, chunkData, spec.chunkSize, numRecipients, auth ?? null);
116
- uploaded += spec.chunkSize;
117
- onProgress?.(uploaded, total);
118
- }
119
- }));
120
- const rcvDescriptions = Array.from({ length: numRecipients }, (_, ri) => buildDescription("recipient", ri, encrypted, sentChunks));
121
- const sndDescription = buildDescription("sender", 0, encrypted, sentChunks);
122
- let uri = encodeDescriptionURI(rcvDescriptions[0]);
123
- let finalRcvDescriptions = rcvDescriptions;
124
- const threshold = redirectThreshold ?? DEFAULT_REDIRECT_THRESHOLD;
125
- if (uri.length > threshold && sentChunks.length > 1) {
126
- const redirected = await uploadRedirectDescription(agent, servers, rcvDescriptions[0], auth);
127
- finalRcvDescriptions = [redirected, ...rcvDescriptions.slice(1)];
128
- uri = encodeDescriptionURI(redirected);
129
- }
130
- return { rcvDescriptions: finalRcvDescriptions, sndDescription, uri };
131
- }
132
- export async function sendFile(agent, servers, source, fileNameOrSize, fileNameOrOptions, maybeOptions) {
133
- let sourceSize, fileName, options;
134
- if (source instanceof Uint8Array) {
135
- sourceSize = source.length;
136
- fileName = fileNameOrSize;
137
- options = fileNameOrOptions;
138
- }
139
- else {
140
- sourceSize = fileNameOrSize;
141
- fileName = fileNameOrOptions;
142
- options = maybeOptions;
143
- }
144
- if (servers.length === 0)
145
- throw new Error("sendFile: servers list is empty");
146
- const { onProgress, auth, numRecipients = 1 } = options ?? {};
147
- const params = prepareEncryption(sourceSize, fileName);
148
- const specs = prepareChunkSpecs(params.chunkSizes);
149
- const total = params.chunkSizes.reduce((a, b) => a + b, 0);
150
- const encState = sbInit(params.key, params.nonce);
151
- const hashState = sha512Init();
152
- const sentChunks = new Array(specs.length);
153
- let specIdx = 0, chunkOff = 0, uploaded = 0;
154
- let chunkBuf = new Uint8Array(specs[0].chunkSize);
155
- async function flushChunk() {
156
- const server = servers[Math.floor(Math.random() * servers.length)];
157
- sentChunks[specIdx] = await uploadSingleChunk(agent, server, specIdx + 1, chunkBuf, specs[specIdx].chunkSize, numRecipients, auth ?? null);
158
- uploaded += specs[specIdx].chunkSize;
58
+ for (let i = 0; i < specs.length; i++) {
59
+ const spec = specs[i];
60
+ const chunkNo = i + 1;
61
+ const sndKp = generateEd25519KeyPair();
62
+ const rcvKp = generateEd25519KeyPair();
63
+ const chunkData = await readChunk(spec.chunkOffset, spec.chunkSize);
64
+ const chunkDigest = getChunkDigest(chunkData);
65
+ console.log(`[AGENT-DBG] upload chunk=${chunkNo} offset=${spec.chunkOffset} size=${spec.chunkSize} digest=${_dbgHex(chunkDigest, 32)} data[0..8]=${_dbgHex(chunkData)} data[-8..]=${_dbgHex(chunkData.slice(-8))}`);
66
+ const fileInfo = {
67
+ sndKey: encodePubKeyEd25519(sndKp.publicKey),
68
+ size: spec.chunkSize,
69
+ digest: chunkDigest
70
+ };
71
+ const rcvKeysForChunk = [encodePubKeyEd25519(rcvKp.publicKey)];
72
+ const { senderId, recipientIds } = await createXFTPChunk(agent, server, sndKp.privateKey, fileInfo, rcvKeysForChunk);
73
+ await uploadXFTPChunk(agent, server, sndKp.privateKey, senderId, chunkData);
74
+ sentChunks.push({
75
+ chunkNo, senderId, senderKey: sndKp.privateKey,
76
+ recipientId: recipientIds[0], recipientKey: rcvKp.privateKey,
77
+ chunkSize: spec.chunkSize, digest: chunkDigest, server
78
+ });
79
+ uploaded += spec.chunkSize;
159
80
  onProgress?.(uploaded, total);
160
- specIdx++;
161
- if (specIdx < specs.length) {
162
- chunkBuf = new Uint8Array(specs[specIdx].chunkSize);
163
- chunkOff = 0;
164
- }
165
- }
166
- async function feedEncrypted(data) {
167
- sha512Update(hashState, data);
168
- let off = 0;
169
- while (off < data.length) {
170
- const space = specs[specIdx].chunkSize - chunkOff;
171
- const n = Math.min(space, data.length - off);
172
- chunkBuf.set(data.subarray(off, off + n), chunkOff);
173
- chunkOff += n;
174
- off += n;
175
- if (chunkOff === specs[specIdx].chunkSize)
176
- await flushChunk();
177
- }
178
81
  }
179
- await feedEncrypted(sbEncryptChunk(encState, concatBytes(encodeInt64(params.fileSize), params.fileHdr)));
180
- const SLICE = 65536;
181
- if (source instanceof Uint8Array) {
182
- for (let off = 0; off < source.length; off += SLICE) {
183
- await feedEncrypted(sbEncryptChunk(encState, source.subarray(off, Math.min(off + SLICE, source.length))));
184
- }
185
- }
186
- else {
187
- for await (const chunk of source) {
188
- for (let off = 0; off < chunk.length; off += SLICE) {
189
- await feedEncrypted(sbEncryptChunk(encState, chunk.subarray(off, Math.min(off + SLICE, chunk.length))));
190
- }
191
- }
192
- }
193
- const padLen = Number(params.encSize - 16n - params.fileSize - 8n);
194
- const padding = new Uint8Array(padLen);
195
- padding.fill(0x23);
196
- await feedEncrypted(sbEncryptChunk(encState, padding));
197
- await feedEncrypted(sbAuth(encState));
198
- const digest = sha512Final(hashState);
199
- const encrypted = { digest, key: params.key, nonce: params.nonce, chunkSizes: params.chunkSizes };
200
- const rcvDescriptions = Array.from({ length: numRecipients }, (_, ri) => buildDescription("recipient", ri, encrypted, sentChunks));
201
- const sndDescription = buildDescription("sender", 0, encrypted, sentChunks);
202
- let uri = encodeDescriptionURI(rcvDescriptions[0]);
203
- let finalRcvDescriptions = rcvDescriptions;
204
- if (uri.length > DEFAULT_REDIRECT_THRESHOLD && sentChunks.length > 1) {
205
- const redirected = await uploadRedirectDescription(agent, servers, rcvDescriptions[0], auth);
206
- finalRcvDescriptions = [redirected, ...rcvDescriptions.slice(1)];
207
- uri = encodeDescriptionURI(redirected);
82
+ const rcvDescription = buildDescription("recipient", encrypted, sentChunks);
83
+ const sndDescription = buildDescription("sender", encrypted, sentChunks);
84
+ let uri = encodeDescriptionURI(rcvDescription);
85
+ let finalRcvDescription = rcvDescription;
86
+ const threshold = redirectThreshold ?? DEFAULT_REDIRECT_THRESHOLD;
87
+ if (uri.length > threshold && sentChunks.length > 1) {
88
+ finalRcvDescription = await uploadRedirectDescription(agent, server, rcvDescription);
89
+ uri = encodeDescriptionURI(finalRcvDescription);
208
90
  }
209
- return { rcvDescriptions: finalRcvDescriptions, sndDescription, uri };
91
+ return { rcvDescription: finalRcvDescription, sndDescription, uri };
210
92
  }
211
- function buildDescription(party, recipientIndex, enc, chunks) {
93
+ function buildDescription(party, enc, chunks) {
212
94
  const defChunkSize = enc.chunkSizes[0];
213
95
  return {
214
96
  party,
@@ -223,38 +105,40 @@ function buildDescription(party, recipientIndex, enc, chunks) {
223
105
  digest: c.digest,
224
106
  replicas: [{
225
107
  server: formatXFTPServer(c.server),
226
- replicaId: party === "recipient" ? c.recipients[recipientIndex].recipientId : c.senderId,
227
- replicaKey: encodePrivKeyEd25519(party === "recipient" ? c.recipients[recipientIndex].recipientKey : c.senderKey)
108
+ replicaId: party === "recipient" ? c.recipientId : c.senderId,
109
+ replicaKey: encodePrivKeyEd25519(party === "recipient" ? c.recipientKey : c.senderKey)
228
110
  }]
229
111
  })),
230
112
  redirect: null
231
113
  };
232
114
  }
233
- async function uploadRedirectDescription(agent, servers, innerFd, auth) {
115
+ async function uploadRedirectDescription(agent, server, innerFd) {
234
116
  const yaml = encodeFileDescription(innerFd);
235
117
  const yamlBytes = new TextEncoder().encode(yaml);
236
- const enc = await encryptFileForUpload(yamlBytes, "");
118
+ const enc = encryptFileForUpload(yamlBytes, "");
237
119
  const specs = prepareChunkSpecs(enc.chunkSizes);
238
- const chunkJobs = specs.map((spec, i) => ({
239
- index: i,
240
- spec,
241
- server: servers[Math.floor(Math.random() * servers.length)]
242
- }));
243
- const byServer = new Map();
244
- for (const job of chunkJobs) {
245
- const key = formatXFTPServer(job.server);
246
- if (!byServer.has(key))
247
- byServer.set(key, []);
248
- byServer.get(key).push(job);
120
+ const sentChunks = [];
121
+ for (let i = 0; i < specs.length; i++) {
122
+ const spec = specs[i];
123
+ const chunkNo = i + 1;
124
+ const sndKp = generateEd25519KeyPair();
125
+ const rcvKp = generateEd25519KeyPair();
126
+ const chunkData = enc.encData.subarray(spec.chunkOffset, spec.chunkOffset + spec.chunkSize);
127
+ const chunkDigest = getChunkDigest(chunkData);
128
+ const fileInfo = {
129
+ sndKey: encodePubKeyEd25519(sndKp.publicKey),
130
+ size: spec.chunkSize,
131
+ digest: chunkDigest
132
+ };
133
+ const rcvKeysForChunk = [encodePubKeyEd25519(rcvKp.publicKey)];
134
+ const { senderId, recipientIds } = await createXFTPChunk(agent, server, sndKp.privateKey, fileInfo, rcvKeysForChunk);
135
+ await uploadXFTPChunk(agent, server, sndKp.privateKey, senderId, chunkData);
136
+ sentChunks.push({
137
+ chunkNo, senderId, senderKey: sndKp.privateKey,
138
+ recipientId: recipientIds[0], recipientKey: rcvKp.privateKey,
139
+ chunkSize: spec.chunkSize, digest: chunkDigest, server
140
+ });
249
141
  }
250
- const sentChunks = new Array(specs.length);
251
- await Promise.all([...byServer.values()].map(async (jobs) => {
252
- for (const { index, spec, server } of jobs) {
253
- const chunkNo = index + 1;
254
- const chunkData = enc.encData.subarray(spec.chunkOffset, spec.chunkOffset + spec.chunkSize);
255
- sentChunks[index] = await uploadSingleChunk(agent, server, chunkNo, chunkData, spec.chunkSize, 1, auth ?? null);
256
- }
257
- }));
258
142
  return {
259
143
  party: "recipient",
260
144
  size: enc.chunkSizes.reduce((a, b) => a + b, 0),
@@ -268,8 +152,8 @@ async function uploadRedirectDescription(agent, servers, innerFd, auth) {
268
152
  digest: c.digest,
269
153
  replicas: [{
270
154
  server: formatXFTPServer(c.server),
271
- replicaId: c.recipients[0].recipientId,
272
- replicaKey: encodePrivKeyEd25519(c.recipients[0].recipientKey)
155
+ replicaId: c.recipientId,
156
+ replicaKey: encodePrivKeyEd25519(c.recipientKey)
273
157
  }]
274
158
  })),
275
159
  redirect: { size: innerFd.size, digest: innerFd.digest }
@@ -279,7 +163,7 @@ export async function downloadFileRaw(agent, fd, onRawChunk, options) {
279
163
  const err = validateFileDescription(fd);
280
164
  if (err)
281
165
  throw new Error("downloadFileRaw: " + err);
282
- const { onProgress } = options ?? {};
166
+ const { onProgress, concurrency = 1 } = options ?? {};
283
167
  // Resolve redirect on main thread (redirect data is small)
284
168
  if (fd.redirect !== null) {
285
169
  console.log(`[AGENT-DBG] resolving redirect: outer size=${fd.size} chunks=${fd.chunks.length}`);
@@ -313,7 +197,6 @@ export async function downloadFileRaw(agent, fd, onRawChunk, options) {
313
197
  body: raw.body,
314
198
  digest: chunk.digest
315
199
  });
316
- await ackXFTPChunk(agent, server, kp.privateKey, replica.replicaId);
317
200
  downloaded += chunk.chunkSize;
318
201
  onProgress?.(downloaded, resolvedFd.size);
319
202
  }
@@ -333,32 +216,18 @@ export async function downloadFile(agent, fd, onProgress) {
333
216
  throw new Error("downloadFile: file digest mismatch");
334
217
  return processDownloadedFile(resolvedFd, chunks);
335
218
  }
336
- export async function receiveFile(agent, uri, options) {
337
- const fd = decodeDescriptionURI(uri);
338
- return downloadFile(agent, fd, options?.onProgress);
339
- }
340
219
  async function resolveRedirect(agent, fd) {
341
220
  const plaintextChunks = new Array(fd.chunks.length);
342
- const byServer = new Map();
343
221
  for (const chunk of fd.chunks) {
344
- const srv = chunk.replicas[0]?.server ?? "";
345
- if (!byServer.has(srv))
346
- byServer.set(srv, []);
347
- byServer.get(srv).push(chunk);
222
+ const replica = chunk.replicas[0];
223
+ if (!replica)
224
+ throw new Error("resolveRedirect: chunk has no replicas");
225
+ const server = parseXFTPServer(replica.server);
226
+ const seed = decodePrivKeyEd25519(replica.replicaKey);
227
+ const kp = ed25519KeyPairFromSeed(seed);
228
+ const data = await downloadXFTPChunk(agent, server, kp.privateKey, replica.replicaId, chunk.digest);
229
+ plaintextChunks[chunk.chunkNo - 1] = data;
348
230
  }
349
- await Promise.all([...byServer.entries()].map(async ([srv, chunks]) => {
350
- const server = parseXFTPServer(srv);
351
- for (const chunk of chunks) {
352
- const replica = chunk.replicas[0];
353
- if (!replica)
354
- throw new Error("resolveRedirect: chunk has no replicas");
355
- const seed = decodePrivKeyEd25519(replica.replicaKey);
356
- const kp = ed25519KeyPairFromSeed(seed);
357
- const data = await downloadXFTPChunk(agent, server, kp.privateKey, replica.replicaId, chunk.digest);
358
- plaintextChunks[chunk.chunkNo - 1] = data;
359
- await ackXFTPChunk(agent, server, kp.privateKey, replica.replicaId);
360
- }
361
- }));
362
231
  const totalSize = plaintextChunks.reduce((s, c) => s + c.length, 0);
363
232
  if (totalSize !== fd.size)
364
233
  throw new Error("resolveRedirect: redirect file size mismatch");
@@ -379,24 +248,15 @@ async function resolveRedirect(agent, fd) {
379
248
  }
380
249
  // -- Delete
381
250
  export async function deleteFile(agent, sndDescription) {
382
- const byServer = new Map();
383
251
  for (const chunk of sndDescription.chunks) {
384
- const srv = chunk.replicas[0]?.server ?? "";
385
- if (!byServer.has(srv))
386
- byServer.set(srv, []);
387
- byServer.get(srv).push(chunk);
252
+ const replica = chunk.replicas[0];
253
+ if (!replica)
254
+ throw new Error("deleteFile: chunk has no replicas");
255
+ const server = parseXFTPServer(replica.server);
256
+ const seed = decodePrivKeyEd25519(replica.replicaKey);
257
+ const kp = ed25519KeyPairFromSeed(seed);
258
+ await deleteXFTPChunk(agent, server, kp.privateKey, replica.replicaId);
388
259
  }
389
- await Promise.all([...byServer.entries()].map(async ([srv, chunks]) => {
390
- const server = parseXFTPServer(srv);
391
- for (const chunk of chunks) {
392
- const replica = chunk.replicas[0];
393
- if (!replica)
394
- throw new Error("deleteFile: chunk has no replicas");
395
- const seed = decodePrivKeyEd25519(replica.replicaKey);
396
- const kp = ed25519KeyPairFromSeed(seed);
397
- await deleteXFTPChunk(agent, server, kp.privateKey, replica.replicaId);
398
- }
399
- }));
400
260
  }
401
261
  // -- Internal
402
262
  function _dbgHex(b, n = 8) {