dsh-dl-server 0.0.0-stage → 0.1.0

package/lib/tools.js

@@ -0,0 +1,216 @@
+import { CLEANUP_SCOPES, DEFAULT_SCOPE, QueueError } from './queue.js'
+import { mark, markFailure } from './bootlog.js'
+
+/**
+ * Turn a domain-level failure into a canonical value instead of an exception.
+ * Only unexpected failures propagate as errors.
+ * @param {unknown} error
+ */
+function fail(error) {
+  if (error instanceof QueueError) {
+    return { ok: false, code: error.code, message: error.message }
+  }
+  throw error
+}
+
+/**
+ * Every DLS tool returns one canonical JSON object.
+ * `ToolDefinition.output` is mandatory and its `render` is mandatory too, so the
+ * shape lives here instead of being repeated seven times.
+ */
+const CANONICAL_OUTPUT = {
+  schema: { type: 'object', additionalProperties: true },
+  render: (_args, value) => [{ type: 'text', text: JSON.stringify(value, null, 2) }],
+}
+
+/**
+ * Best-effort caller identity. DSH's tool execution context is the only source;
+ * when a field is missing we fall back to a stable placeholder so the queue
+ * still behaves sanely in a single-session profile.
+ * @param {any} exec
+ */
+export function resolveCaller(exec) {
+  const agent = exec?.agent
+  const session =
+    agent?.sessionId ?? agent?.session?.id ?? exec?.session?.id ?? exec?.sessionId ?? 'local'
+  const workspace =
+    agent?.workspaceId ?? agent?.workspace?.id ?? exec?.workspaceId ?? 'local'
+  return { workspace: String(workspace), session: String(session) }
+}
+
+/**
+ * Register the seven DLS tools.
+ *
+ * @param {object} deps
+ * @param {any} deps.ctx
+ * @param {import('./queue.js').Queue} deps.queue
+ * @param {object} deps.config
+ */
+export function registerTools({ ctx, queue, config }) {
+  mark('registerTools enter')
+
+  ctx.inject(['tools'], (scope) => {
+    try {
+      mark('tools inject fired', `hasTools=${typeof scope?.tools?.register === 'function'}`)
+      const tools = scope.tools
+      const withCaller = (exec) => resolveCaller(exec)
+
+      tools.register({
+        name: 'download_enqueue',
+        output: CANONICAL_OUTPUT,
+        description:
+          'Queue a file for download without blocking. Returns a job id such as J#1042 immediately; ' +
+          'use download_wait or download_status to follow it. The job survives the turn, the session and ' +
+          'a DSH restart. Every session can see every job, but only the owning session can pause, resume ' +
+          'or cancel an in-flight one.',
+        parameters: {
+          type: 'object',
+          properties: {
+            urls: {
+              type: 'array',
+              items: { type: 'string' },
+              minItems: 1,
+              description: 'Candidate HTTP(S) URLs, tried in order. The first that answers wins.',
+            },
+            dest: {
+              type: 'string',
+              description: 'Absolute destination path including the file name.',
+            },
+            sha256: {
+              type: 'string',
+              description: 'Optional expected SHA-256. The job fails if the finished file does not match.',
+            },
+          },
+          required: ['urls', 'dest'],
+          additionalProperties: false,
+        },
+        async execute(args, exec) {
+          try {
+            const job = await queue.enqueue(
+              { urls: args.urls, destPath: args.dest, expectSha256: args.sha256 },
+              withCaller(exec),
+            )
+            return { ok: true, jobId: job.id, state: job.state, dest: job.dest }
+          } catch (error) {
+            return fail(error)
+          }
+        },
+      })
+
+      tools.register({
+        name: 'download_status',
+        output: CANONICAL_OUTPUT,
+        description:
+          'List DLS jobs. Every session sees every job; the owner field shows which session owns each one. ' +
+          'Omit id to list all jobs, pass an id such as J#1042 for a single job.',
+        parameters: {
+          type: 'object',
+          properties: {
+            id: { type: 'string', description: 'Optional job id. Omit to list every job.' },
+          },
+          additionalProperties: false,
+        },
+        async execute(args) {
+          if (args?.id) {
+            const job = queue.get(args.id)
+            return job
+              ? { ok: true, count: 1, jobs: [job] }
+              : { ok: false, code: 'not-found', message: `${args.id} does not exist` }
+          }
+          const jobs = queue.list()
+          return { ok: true, count: jobs.length, jobs }
+        },
+      })
+
+      tools.register({
+        name: 'download_wait',
+        output: CANONICAL_OUTPUT,
+        description:
+          'Block until a job reaches a terminal state (completed / failed / canceled) or the timeout expires. ' +
+          'Returns the current job view either way; check state to tell which happened.',
+        parameters: {
+          type: 'object',
+          properties: {
+            id: { type: 'string', description: 'Job id such as J#1042.' },
+            timeoutSeconds: {
+              type: 'number',
+              description: 'How long to wait. Default 60, maximum 600.',
+            },
+          },
+          required: ['id'],
+          additionalProperties: false,
+        },
+        async execute(args) {
+          const seconds = Math.min(Math.max(Number(args.timeoutSeconds ?? 60), 1), 600)
+          try {
+            const job = await queue.wait(args.id, seconds * 1000)
+            return { ok: true, job }
+          } catch (error) {
+            return fail(error)
+          }
+        },
+      })
+
+      for (const [toolName, method, verb] of [
+        ['download_pause', 'pause', 'Pause'],
+        ['download_resume', 'resume', 'Resume'],
+        ['download_cancel', 'cancel', 'Cancel'],
+      ]) {
+        tools.register({
+          name: toolName,
+          output: CANONICAL_OUTPUT,
+          description:
+            `${verb} a DLS job. Only the session that enqueued the job may do this while it is in flight; ` +
+            'finished jobs are open to everyone. A paused job keeps its partial file and can resume later.',
+          parameters: {
+            type: 'object',
+            properties: { id: { type: 'string', description: 'Job id such as J#1042.' } },
+            required: ['id'],
+            additionalProperties: false,
+          },
+          async execute(args, exec) {
+            try {
+              const job = await queue[method](args.id, withCaller(exec))
+              return { ok: true, job }
+            } catch (error) {
+              return fail(error)
+            }
+          },
+        })
+      }
+
+      tools.register({
+        name: 'download_cleanup',
+        output: CANONICAL_OUTPUT,
+        description:
+          'Clean up the calling session\'s DLS records and leftovers. Default scope "allWithoutFiles" removes ' +
+          'finished records and partial files but never touches the downloaded files themselves. Passing "all" ' +
+          'also deletes those files and cannot be undone.',
+        parameters: {
+          type: 'object',
+          properties: {
+            scope: {
+              type: 'string',
+              enum: CLEANUP_SCOPES,
+              description: `Default ${DEFAULT_SCOPE}. "all" is the only value that deletes downloaded files.`,
+            },
+          },
+          additionalProperties: false,
+        },
+        async execute(args, exec) {
+          try {
+            const result = await queue.cleanup(args?.scope ?? DEFAULT_SCOPE, withCaller(exec))
+            return { ok: true, ...result }
+          } catch (error) {
+            return fail(error)
+          }
+        },
+      })
+
+      mark('tools registered', '7/7')
+    } catch (error) {
+      markFailure('registerTools', error)
+      throw error
+    }
+  })
+}

package/lib/transport.js

@@ -0,0 +1,357 @@
+import { spawn } from 'node:child_process'
+import { createWriteStream } from 'node:fs'
+import fs from 'node:fs/promises'
+import os from 'node:os'
+import path from 'node:path'
+import { Readable } from 'node:stream'
+import { pipeline } from 'node:stream/promises'
+import { randomBytes } from 'node:crypto'
+
+/**
+ * Transport layer. Two channels: an aria2c JSON-RPC daemon when one is
+ * available, and a plain Node fetch stream otherwise.
+ *
+ * The queue never talks to aria2 directly; it calls {@link Transport#transfer}.
+ *
+ * @typedef {object} TransferOptions
+ * @property {string[]} urls            candidate sources, tried in order
+ * @property {string} destPath          absolute final file path
+ * @property {number} [connections]     aria2 split, default 16
+ * @property {number} [speedLimitKbps]  0 or absent means unlimited
+ * @property {string} [proxy]           e.g. http://127.0.0.1:65532
+ * @property {number} [bytesDone]       resume hint for the direct channel
+ * @property {(done: number, total: number) => void} [onProgress]
+ * @property {AbortSignal} [signal]
+ *
+ * @typedef {object} TransferResult
+ * @property {number} bytes
+ * @property {number} total             -1 when the server never reported one
+ */
+
+const IS_WIN = process.platform === 'win32'
+const ARIA2_BIN = IS_WIN ? 'aria2c.exe' : 'aria2c'
+
+/** Absolute locations worth checking beyond PATH. */
+function extraCandidates() {
+  if (IS_WIN) {
+    const local = process.env.LOCALAPPDATA ?? ''
+    const pf = process.env.ProgramFiles ?? 'C:\\Program Files'
+    const pf86 = process.env['ProgramFiles(x86)'] ?? 'C:\\Program Files (x86)'
+    return [
+      path.join(pf, 'aria2', ARIA2_BIN),
+      path.join(pf86, 'aria2', ARIA2_BIN),
+      path.join(local, 'Programs', 'Motrix', 'resources', 'aria2', ARIA2_BIN),
+      path.join(local, 'Motrix', 'resources', 'aria2', ARIA2_BIN),
+    ]
+  }
+  return ['/usr/bin/aria2c', '/usr/local/bin/aria2c', '/opt/homebrew/bin/aria2c']
+}
+
+async function isExecutable(file) {
+  try {
+    await fs.access(file, fs.constants.X_OK)
+    return true
+  } catch {
+    return false
+  }
+}
+
+/** @param {string} name */
+async function findInPath(name) {
+  const dirs = (process.env.PATH ?? '').split(path.delimiter).filter(Boolean)
+  for (const dir of dirs) {
+    const candidate = path.join(dir, name)
+    if (await isExecutable(candidate)) return candidate
+  }
+  return undefined
+}
+
+/**
+ * Locate an aria2c binary.
+ * Order: explicit config, environment, PATH, well-known install locations.
+ *
+ * @param {string} [configured]
+ * @returns {Promise<string | undefined>}
+ */
+export async function findAria2(configured) {
+  if (configured) {
+    try {
+      await fs.access(configured, fs.constants.X_OK)
+      return configured
+    } catch {
+      /* fall through */
+    }
+  }
+
+  for (const key of ['DLS_ARIA2C_PATH', 'MT_ARIA2C_PATH', 'ARIA2C_PATH']) {
+    const value = process.env[key]
+    if (value) {
+      try {
+        await fs.access(value, fs.constants.X_OK)
+        return value
+      } catch {
+        /* fall through */
+      }
+    }
+  }
+
+  const fromPath = await findInPath(ARIA2_BIN)
+  if (fromPath) return fromPath
+
+  for (const candidate of extraCandidates()) {
+    if (await isExecutable(candidate)) return candidate
+  }
+  return undefined
+}
+
+/** Minimal JSON-RPC client for the aria2 daemon. */
+class Aria2Rpc {
+  /** @param {number} port @param {string} secret */
+  constructor(port, secret) {
+    this.port = port
+    this.secret = secret
+    this.url = `http://127.0.0.1:${port}/jsonrpc`
+    this.nextId = 1
+  }
+
+  /** @param {string} method @param {unknown[]} params */
+  async call(method, params = []) {
+    const body = JSON.stringify({
+      jsonrpc: '2.0',
+      id: this.nextId++,
+      method,
+      params: [`token:${this.secret}`, ...params],
+    })
+    const response = await fetch(this.url, {
+      method: 'POST',
+      headers: { 'content-type': 'application/json' },
+      body,
+    })
+    const payload = await response.json()
+    if (payload.error) {
+      throw new Error(`aria2 ${method}: ${payload.error.message}`)
+    }
+    return payload.result
+  }
+
+  /** Poll until the daemon answers, or give up. */
+  async waitReady(timeoutMs = 8000) {
+    const deadline = Date.now() + timeoutMs
+    while (Date.now() < deadline) {
+      try {
+        await this.call('aria2.getVersion')
+        return true
+      } catch {
+        await sleep(120)
+      }
+    }
+    return false
+  }
+}
+
+function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms))
+}
+
+/**
+ * The transport facade. One instance per plugin; the daemon is shared by every
+ * concurrent job and torn down on dispose.
+ */
+export class Transport {
+  /**
+   * @param {{ aria2Path?: string, aria2Port?: number, connectionsPerJob?: number,
+   *           speedLimitKbps?: number, proxy?: string }} config
+   * @param {(level: string, message: string) => void} [log]
+   */
+  constructor(config = {}, log = () => {}) {
+    this.config = config
+    this.log = log
+    this.mode = 'direct'
+    this.aria2 = undefined
+    this.rpc = undefined
+  }
+
+  /** Detect aria2 and start the daemon if possible. Never throws. */
+  async init() {
+    const binary = await findAria2(this.config.aria2Path)
+    if (!binary) {
+      this.log('info', 'aria2c not found, using the single-stream fallback')
+      return
+    }
+
+    const port = this.config.aria2Port ?? 16801
+    const secret = randomBytes(12).toString('hex')
+    const args = [
+      '--enable-rpc',
+      `--rpc-listen-port=${port}`,
+      '--rpc-listen-all=false',
+      `--rpc-secret=${secret}`,
+      '--continue=true',
+      '--auto-file-renaming=false',
+      '--allow-overwrite=true',
+      '--log-level=warn',
+      '--console-log-level=warn',
+      '--summary-interval=0',
+      '--disable-ipv6=true',
+    ]
+    if (this.config.proxy) args.push(`--all-proxy=${this.config.proxy}`)
+
+    try {
+      this.aria2 = spawn(binary, args, { windowsHide: true, stdio: 'ignore' })
+      this.aria2.unref()
+      this.aria2.on('exit', () => {
+        this.aria2 = undefined
+        this.mode = 'direct'
+      })
+
+      const rpc = new Aria2Rpc(port, secret)
+      if (await rpc.waitReady()) {
+        this.rpc = rpc
+        this.mode = 'aria2'
+        this.log('info', `aria2c ready on port ${port} (${binary})`)
+      } else {
+        this.log('warn', 'aria2c started but never answered; falling back')
+        this.aria2?.kill()
+        this.aria2 = undefined
+      }
+    } catch (error) {
+      this.log('warn', `aria2c failed to start: ${error.message}`)
+      this.mode = 'direct'
+    }
+  }
+
+  /** @param {TransferOptions} options @returns {Promise<TransferResult>} */
+  async transfer(options) {
+    if (this.mode === 'aria2' && this.rpc) {
+      try {
+        return await this.#transferAria2(options)
+      } catch (error) {
+        if (options.signal?.aborted) throw error
+        this.log('warn', `aria2 transfer failed, retrying single-stream: ${error.message}`)
+      }
+    }
+    return this.#transferDirect(options)
+  }
+
+  /** @param {TransferOptions} options */
+  async #transferAria2({ urls, destPath, connections, speedLimitKbps, onProgress, signal }) {
+    const rpc = this.rpc
+    await fs.mkdir(path.dirname(destPath), { recursive: true })
+
+    const options = {
+      dir: path.dirname(destPath),
+      out: path.basename(destPath),
+      split: String(connections ?? this.config.connectionsPerJob ?? 16),
+      'max-connection-per-server': String(connections ?? this.config.connectionsPerJob ?? 16),
+      'min-split-size': '1M',
+      continue: 'true',
+      'auto-file-renaming': 'false',
+      'allow-overwrite': 'true',
+    }
+    const limit = speedLimitKbps ?? this.config.speedLimitKbps ?? 0
+    if (limit > 0) options['max-download-limit'] = `${limit}K`
+
+    const gid = await rpc.call('aria2.addUri', [urls, options])
+
+    let aborted = false
+    const onAbort = () => {
+      aborted = true
+      void rpc.call('aria2.remove', [gid]).catch(() => {})
+    }
+    signal?.addEventListener('abort', onAbort, { once: true })
+
+    try {
+      for (;;) {
+        if (aborted) throw new Error('canceled')
+        const status = await rpc.call('aria2.tellStatus', [
+          gid,
+          ['status', 'totalLength', 'completedLength', 'errorMessage'],
+        ])
+        const done = Number(status.completedLength ?? 0)
+        const total = status.totalLength === '0' ? -1 : Number(status.totalLength)
+        onProgress?.(done, total)
+
+        if (status.status === 'complete') return { bytes: done, total }
+        if (status.status === 'error' || status.status === 'removed') {
+          throw new Error(status.errorMessage || `aria2 status ${status.status}`)
+        }
+        if (status.status === 'paused') return { bytes: done, total }
+        await sleep(400)
+      }
+    } finally {
+      signal?.removeEventListener('abort', onAbort)
+    }
+  }
+
+  /** @param {TransferOptions} options */
+  async #transferDirect({ urls, destPath, bytesDone = 0, onProgress, signal }) {
+    await fs.mkdir(path.dirname(destPath), { recursive: true })
+    const partPath = `${destPath}.part`
+    let lastError
+
+    for (const url of urls) {
+      try {
+        let offset = bytesDone
+        try {
+          offset = (await fs.stat(partPath)).size
+        } catch {
+          offset = bytesDone
+        }
+
+        const headers = {}
+        if (offset > 0) headers.range = `bytes=${offset}-`
+
+        const response = await fetch(url, { headers, signal })
+        if (!response.ok) throw new Error(`HTTP ${response.status}`)
+
+        const ranged = response.status === 206
+        const start = ranged ? offset : 0
+        const declared = Number(response.headers.get('content-length') ?? 0)
+        const total = declared > 0 ? declared + start : -1
+
+        const sink = createWriteStream(partPath, { flags: ranged ? 'a' : 'w' })
+        let written = start
+        let lastTick = 0
+
+        const source = Readable.fromWeb(response.body)
+        source.on('data', (chunk) => {
+          written += chunk.length
+          const now = Date.now()
+          if (now - lastTick > 400) {
+            lastTick = now
+            onProgress?.(written, total)
+          }
+        })
+
+        await pipeline(source, sink)
+        onProgress?.(written, total)
+        await fs.rename(partPath, destPath)
+        return { bytes: written, total }
+      } catch (error) {
+        if (error?.name === 'AbortError' || signal?.aborted) throw error
+        lastError = error
+      }
+    }
+    throw lastError ?? new Error('no source available')
+  }
+
+  /** Stop the daemon. Safe to call twice. */
+  async dispose() {
+    try {
+      await this.rpc?.call('aria2.shutdown')
+    } catch {
+      /* daemon already gone */
+    }
+    this.rpc = undefined
+    this.mode = 'direct'
+    try {
+      this.aria2?.kill()
+    } catch {
+      /* ignore */
+    }
+    this.aria2 = undefined
+  }
+}
+
+/** Exposed for tests: resolve a candidate without starting anything. */
+export const _internal = { findInPath, extraCandidates, Aria2Rpc, sleep, homeDir: os.homedir }

package/lib/types.js

@@ -0,0 +1,54 @@
+/**
+ * DLS data model. Zero dependencies, plain JS.
+ *
+ * @typedef {'queued'|'running'|'paused'|'completed'|'failed'|'canceled'} JobState
+ * @typedef {'user'|'session_closed'|'host_restart'} PauseReason
+ *
+ * @typedef {object} JobRecord
+ * @property {string} id             `J#1042`
+ * @property {number} seq            monotonic sequence, matches the number in id
+ * @property {string} ownerWorkspace WorkspaceId (uuid)
+ * @property {string} ownerSession   SessionId (uuid) of the parent session
+ * @property {string} [origin]       subagent name, display and audit only
+ * @property {string[]} urls
+ * @property {string} destPath
+ * @property {JobState} state
+ * @property {PauseReason} [pauseReason]
+ * @property {number} createdAt
+ * @property {number} updatedAt
+ * @property {number} bytesTotal     -1 when unknown
+ * @property {number} bytesCheckpoint  checkpoint bytes, not live progress
+ * @property {number} attempts
+ * @property {string} [lastError]
+ * @property {string} [expectSha256]
+ * @property {number} [finalBytes]
+ */
+
+export const JOB_STATES = Object.freeze([
+  'queued',
+  'running',
+  'paused',
+  'completed',
+  'failed',
+  'canceled',
+])
+
+const TERMINAL = new Set(['completed', 'failed', 'canceled'])
+
+/** @param {JobState} state */
+export function isTerminal(state) {
+  return TERMINAL.has(state)
+}
+
+/** @param {number} seq */
+export function formatJobId(seq) {
+  return `J#${seq}`
+}
+
+const JOB_ID_RE = /^J#(\d+)$/
+
+/** @param {string} id @returns {number | undefined} */
+export function parseJobId(id) {
+  const m = JOB_ID_RE.exec(id)
+  return m ? Number(m[1]) : undefined
+}

package/package.json

@@ -1,6 +1,57 @@
 {
   "name": "dsh-dl-server",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
-}
+  "version": "0.1.0",
+  "description": "DLS — DSH Download Server: a global, session-aware, persistent download queue for DeepSeek Harness.",
+  "type": "module",
+  "main": "lib/index.js",
+  "exports": {
+    ".": "./lib/index.js",
+    "./cordis.patch.yml": "./cordis.patch.yml",
+    "./package.json": "./package.json"
+  },
+  "files": [
+    "lib",
+    "cordis.patch.yml",
+    "LICENSE",
+    "README.md",
+    "README.en.md"
+  ],
+  "keywords": [
+    "deepseek-harness",
+    "dsh",
+    "dsh-plugin",
+    "download",
+    "downloader",
+    "queue",
+    "aria2",
+    "dls"
+  ],
+  "author": "Jestions",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/Jestions-dev/dsh-download-server.git"
+  },
+  "homepage": "https://github.com/Jestions-dev/dsh-download-server#readme",
+  "bugs": {
+    "url": "https://github.com/Jestions-dev/dsh-download-server/issues"
+  },
+  "engines": {
+    "node": ">=20"
+  },
+  "scripts": {
+    "test": "node --test",
+    "check": "node --check lib/index.js && node --check lib/domain.js && node --check lib/types.js"
+  },
+  "dependencies": {
+    "@deepseek-ai/cordis": "~4.1.0",
+    "@deepseek-ai/dsh-storage": "^0.2.2-rc.1",
+    "@deepseek-ai/dsh-storage-domain": "^0.2.2-rc.1",
+    "zod": "^4.4.3"
+  },
+  "dsh": {
+    "bundle": {
+      "patch": "./cordis.patch.yml"
+    }
+  }
+}