dsh-dl-server 0.1.1 → 0.3.0

package/lib/tools.js

@@ -90,7 +90,13 @@ export function registerTools({ ctx, queue, config }) {
               { urls: args.urls, destPath: args.dest, expectSha256: args.sha256 },
               withCaller(exec),
             )
-            return { ok: true, jobId: job.id, state: job.state, dest: job.dest }
+            return {
+              ok: true,
+              jobId: job.id,
+              state: job.state,
+              dest: job.dest,
+              ...(job.deduplicated ? { deduplicated: true } : {}),
+            }
           } catch (error) {
             return fail(error)
           }
@@ -111,14 +117,15 @@ export function registerTools({ ctx, queue, config }) {
           additionalProperties: false,
         },
         async execute(args) {
+          const health = queue.health()
           if (args?.id) {
             const job = queue.get(args.id)
             return job
-              ? { ok: true, count: 1, jobs: [job] }
+              ? { ok: true, count: 1, jobs: [job], ...health }
               : { ok: false, code: 'not-found', message: `${args.id} does not exist` }
           }
           const jobs = queue.list()
-          return { ok: true, count: jobs.length, jobs }
+          return { ok: true, count: jobs.length, jobs, ...health }
         },
       })
 

package/lib/transport.js

@@ -6,6 +6,7 @@ import path from 'node:path'
 import { Readable } from 'node:stream'
 import { pipeline } from 'node:stream/promises'
 import { randomBytes } from 'node:crypto'
+import { sameResource } from './mirrors.js'
 
 /**
  * Transport layer. Two channels: an aria2c JSON-RPC daemon when one is
@@ -16,10 +17,13 @@ import { randomBytes } from 'node:crypto'
  * @typedef {object} TransferOptions
  * @property {string[]} urls            candidate sources, tried in order
  * @property {string} destPath          absolute final file path
- * @property {number} [connections]     aria2 split, default 16
+ * @property {number} [connections]     aria2 split for a large file, default 16
+ * @property {number} [connectionsCap]  upper bound when the size picks the count
+ * @property {boolean} [preferDirect]   skip aria2 for this job
  * @property {number} [speedLimitKbps]  0 or absent means unlimited
  * @property {string} [proxy]           e.g. http://127.0.0.1:65532
  * @property {number} [bytesDone]       resume hint for the direct channel
+ * @property {(url: string, ok: boolean) => void} [onUrlResult]
  * @property {(done: number, total: number) => void} [onProgress]
  * @property {AbortSignal} [signal]
  *
@@ -28,6 +32,28 @@ import { randomBytes } from 'node:crypto'
  * @property {number} total             -1 when the server never reported one
  */
 
+/** Below this, aria2's setup cost exceeds any segmentation gain. */
+export const SMALL_FILE_BYTES = 1 * 1024 * 1024
+/** From this size up, extra connections pay for themselves. */
+export const LARGE_FILE_BYTES = 8 * 1024 * 1024
+
+/**
+ * Decide the channel and the number of connections from the file size.
+ *
+ * Segmentation only helps once `min-split-size=1M` can produce several pieces,
+ * and too many connections on a per-IP-throttled host hurts rather than helps.
+ * @param {number} sizeBytes
+ * @param {number} [cap]
+ * @returns {{ connections: number, direct: boolean }}
+ */
+export function planTransfer(sizeBytes, cap = 16) {
+  if (!Number.isFinite(sizeBytes) || sizeBytes <= 0) return { connections: Math.min(cap, 8), direct: false }
+  if (sizeBytes < SMALL_FILE_BYTES) return { connections: 1, direct: true }
+  if (sizeBytes < LARGE_FILE_BYTES) return { connections: Math.min(4, cap), direct: false }
+  return { connections: cap, direct: false }
+}
+
+
 const IS_WIN = process.platform === 'win32'
 const ARIA2_BIN = IS_WIN ? 'aria2c.exe' : 'aria2c'
 
@@ -169,6 +195,8 @@ export class Transport {
     this.mode = 'direct'
     this.aria2 = undefined
     this.rpc = undefined
+    /** @type {Map<string, { verified: string[], rejected: { url: string, reason: string }[] }>} */
+    this.mirrorReports = new Map()
   }
 
   /** Detect aria2 and start the daemon if possible. Never throws. */
@@ -222,15 +250,88 @@ export class Transport {
 
   /** @param {TransferOptions} options @returns {Promise<TransferResult>} */
   async transfer(options) {
-    if (this.mode === 'aria2' && this.rpc) {
+    const origin = await this.#probeHead(options.urls[0], options.signal)
+    const plan = planTransfer(origin.length, options.connectionsCap ?? options.connections ?? this.config.connectionsPerJob ?? 16)
+    const connections = options.connections ?? plan.connections
+
+    const urls = await this.#verifiedUrls(options.urls, origin, options.signal)
+    const useAria2 = this.mode === 'aria2' && this.rpc && !options.preferDirect && !plan.direct
+
+    if (useAria2) {
       try {
-        return await this.#transferAria2(options)
+        const result = await this.#transferAria2({ ...options, urls, connections })
+        options.onUrlResult?.(urls[0], true)
+        return result
       } catch (error) {
         if (options.signal?.aborted) throw error
         this.log('warn', `aria2 transfer failed, retrying single-stream: ${error.message}`)
       }
     }
-    return this.#transferDirect(options)
+    return this.#transferDirect({ ...options, urls })
+  }
+
+  /**
+   * Keep only the mirrors that can be shown to serve the same bytes as the
+   * origin. A mirror is a third party: it may hold an older build or answer with
+   * an HTML error page, and `sha256` is the only proof of content. Length plus
+   * ETag catches the common mistakes; the reasoning is documented in SPEC §11.2.
+   * @param {string[]} urls
+   * @param {{ length: number, etag?: string }} origin
+   * @param {AbortSignal} [signal]
+   */
+  async #verifiedUrls(urls, origin, signal) {
+    if (urls.length <= 1) return urls
+
+    const accepted = [urls[0]]
+    /** @type {{ url: string, reason: string }[]} */
+    const rejected = []
+
+    for (const mirror of urls.slice(1)) {
+      const head = await this.#probeHead(mirror, signal)
+      const verdict = sameResource(origin, head)
+      if (verdict.ok) {
+        accepted.push(mirror)
+        if (verdict.reason === 'origin-unknown') {
+          this.log('warn', `mirror ${mirror} kept unverified: the origin did not report a length`)
+        }
+      } else {
+        rejected.push({ url: mirror, reason: verdict.reason ?? 'unverified' })
+        this.log('warn', `mirror ${mirror} rejected: ${verdict.reason}`)
+      }
+    }
+
+    this.mirrorReports.set(urls[0], { verified: accepted.slice(1), rejected })
+    return accepted
+  }
+
+  /**
+   * Best-effort HEAD: the length and validator used for sizing and for proving a
+   * mirror matches. A failure is not an error — unknown size only means the
+   * planner stays conservative.
+   * @param {string} url
+   * @param {AbortSignal} [signal]
+   * @returns {Promise<{ length: number, etag?: string }>}
+   */
+  async #probeHead(url, signal) {
+    if (!url) return { length: -1 }
+    const controller = new AbortController()
+    const timer = setTimeout(() => controller.abort(), 5000)
+    timer.unref?.()
+    const onAbort = () => controller.abort()
+    signal?.addEventListener('abort', onAbort, { once: true })
+
+    try {
+      const response = await fetch(url, { method: 'HEAD', signal: controller.signal })
+      if (!response.ok) return { length: -1 }
+      const length = Number(response.headers.get('content-length') ?? 0)
+      const etag = response.headers.get('etag') ?? undefined
+      return { length: Number.isFinite(length) && length > 0 ? length : -1, etag }
+    } catch {
+      return { length: -1 }
+    } finally {
+      clearTimeout(timer)
+      signal?.removeEventListener('abort', onAbort)
+    }
   }
 
   /** @param {TransferOptions} options */
@@ -284,7 +385,7 @@ export class Transport {
   }
 
   /** @param {TransferOptions} options */
-  async #transferDirect({ urls, destPath, bytesDone = 0, onProgress, signal }) {
+  async #transferDirect({ urls, destPath, bytesDone = 0, onProgress, onUrlResult, signal }) {
     await fs.mkdir(path.dirname(destPath), { recursive: true })
     const partPath = `${destPath}.part`
     let lastError
@@ -326,10 +427,12 @@ export class Transport {
         await pipeline(source, sink)
         onProgress?.(written, total)
         await fs.rename(partPath, destPath)
+        onUrlResult?.(url, true)
         return { bytes: written, total }
       } catch (error) {
         if (error?.name === 'AbortError' || signal?.aborted) throw error
         lastError = error
+        onUrlResult?.(url, false)
       }
     }
     throw lastError ?? new Error('no source available')

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dsh-dl-server",
-  "version": "0.1.1",
+  "version": "0.3.0",
   "description": "DLS — DSH Download Server: a global, session-aware, persistent download queue for DeepSeek Harness.",
   "type": "module",
   "main": "lib/index.js",