@jaco-tech/dsh-web-fetch-crw 0.1.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT
+
+Copyright (c) 2026 Jaco
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,93 @@
+# dsh-web-fetch-crw
+
+[![dsh-plugin](https://img.shields.io/badge/dsh-plugin-blue)](https://github.com/topics/dsh-plugin)
+[![CI/CD](https://github.com/jaco-tech/dsh-web-fetch-crw/actions/workflows/ci.yml/badge.svg)](https://github.com/jaco-tech/dsh-web-fetch-crw/actions/workflows/ci.yml)
+
+[crw](https://github.com/us/crw) (Firecrawl-compatible) fetch provider plugin for [DeepSeek Harness](https://github.com/deepseek-ai/deepseek-harness).
+
+Registers a `WebFetchProvider` into `ctx.web` so that the existing `web_fetch` model tool retrieves page content through your [crw](https://github.com/us/crw) instance — which converts HTML to clean markdown. No auth, no API key.
+
+crw is a self-hosted web scraping and crawling service that implements the [Firecrawl](https://github.com/nicknisi/firecrawl) API. It provides a drop-in replacement for the Firecrawl `/v1/scrape` endpoint.
+
+## Status
+
+⚠️ **Not yet published to npm.** The CI/CD workflow builds on every push and publishes automatically when a `v*` tag is pushed. Once the first release is cut:
+
+```bash
+npm install @jaco-tech/dsh-web-fetch-crw
+```
+
+## Configure
+
+Add to your DSH profile's `cordis.patch.yml`:
+
+```yaml
+- insert:
+    - id: web-fetch-crw
+      name: '@jaco-tech/dsh-web-fetch-crw'
+      config:
+        baseURL: 'http://your-crw-instance:3000'
+```
+
+Optionally pin `ctx.web` to use this provider explicitly:
+
+```yaml
+- replace:
+    - id: web
+      config:
+        fetchProvider: crw
+```
+
+## How it works
+
+This is a [Cordis](https://github.com/cordiverse/cordis) plugin that implements the `WebFetchProvider` interface from `@deepseek-ai/dsh-web`:
+
+```typescript
+export const name = "web-fetch-crw"
+export const inject = ["web"]
+
+export function apply(ctx, config) {
+  ctx.web.registerFetchProvider({
+    id: "crw",
+    available() { /* URL.canParse(config.baseURL) */ },
+    async fetch(request, signal) {
+      // POST /v1/scrape { url, formats: ["markdown"] }
+      // maps response to WebFetchResult (body as markdown text)
+    }
+  })
+}
+```
+
+The existing `@deepseek-ai/dsh-tool-web` consumer calls `ctx.web.fetch()` — your crw instance serves the content automatically.
+
+## Provider interface
+
+| Method | Behavior |
+|---|---|
+| `available()` | Returns `true` when `baseURL` is a valid URL |
+| `fetch(request, signal?)` | `POST /v1/scrape` with `{ url, formats: ["markdown"] }`. Returns the scraped markdown as `kind: "text"` body (no double-conversion through turndown). |
+
+## Upstream
+
+This plugin communicates with [crw](https://github.com/us/crw) via its Firecrawl-compatible API (`POST /v1/scrape`). crw is a self-hosted web scraping and crawling service that renders pages (via Lightpanda or Chromium) and produces clean markdown. You need a running crw instance to use this plugin.
+
+## Release process
+
+```bash
+# Tag and push — the CI workflow builds and publishes to npm automatically
+git tag v0.1.0
+git push origin v0.1.0
+```
+
+Requires the `NPM_TOKEN` secret to be set in the repository.
+
+## Development
+
+```bash
+npm install
+npm run build
+```
+
+## License
+
+MIT

package/lib/index.js

@@ -0,0 +1,138 @@
+import z from "@deepseek-ai/schemastery";
+import { WebError } from "@deepseek-ai/dsh-web";
+//#endregion
+//#region provider
+/**
+ * Stable id this fetch provider registers under.
+ */
+const CRW_PROVIDER_ID = "crw";
+/**
+ * Map a crw scrape response to the seam's normalized `WebFetchResult`.
+ * crw returns markdown through `data.markdown` (Firecrawl-compatible API).
+ */
+function mapCrwResponse(url, body) {
+    if (!body.success) {
+        const message = body.error ?? body.message ?? "crw scrape returned success: false";
+        throw new WebError(`crw scrape failed: ${message}`, "WEB_PROVIDER_ERROR");
+    }
+    const data = body.data ?? {};
+    const markdown = data.markdown ?? data.raw ?? "";
+    const metadata = data.metadata ?? {};
+    const effectiveUrl = metadata.sourceURL ?? url;
+    const statusCode = metadata.statusCode ?? 200;
+    // crw already returns markdown — send as 'text' to avoid double-conversion
+    // through turndown in dsh-tool-web.
+    const truncated = data.truncated === true;
+    return {
+        url: effectiveUrl,
+        statusCode,
+        body: { kind: "text", content: markdown },
+        truncated,
+    };
+}
+/**
+ * The crw-backed fetch provider. Each call issues a POST to `/v1/scrape`
+ * with the target URL and receives markdown content back.
+ *
+ * crw is a Firecrawl-compatible crawl/scrape service. No auth, no API key.
+ */
+class CrwFetchProvider {
+    resolveOptions;
+    id = CRW_PROVIDER_ID;
+    constructor(resolveOptions) {
+        this.resolveOptions = resolveOptions;
+    }
+    available() {
+        const options = this.resolveOptions();
+        return URL.canParse(options.baseURL);
+    }
+    async fetch(request, signal) {
+        const options = this.resolveOptions();
+        throwIfAborted(signal);
+        const endpoint = `${options.baseURL}/v1/scrape`;
+        throwIfAborted(signal);
+        let response;
+        try {
+            response = await fetch(endpoint, {
+                method: "POST",
+                headers: {
+                    "content-type": "application/json",
+                    accept: "application/json",
+                    "user-agent": "deepseek-harness/0.0.1",
+                },
+                body: JSON.stringify({ url: request.url, formats: ["markdown"] }),
+                ...signal !== void 0 ? { signal } : {},
+            });
+        }
+        catch (error) {
+            if (signal?.aborted === true || isAbortError(error))
+                throw aborted(signal, error);
+            throw new WebError(`crw fetch request failed: ${String(error)}`, "WEB_PROVIDER_ERROR", { cause: error });
+        }
+        if (!response.ok) {
+            let message = `crw API error (HTTP ${response.status})`;
+            try {
+                const parsed = (await response.json());
+                if (parsed.error != null && parsed.error.length > 0)
+                    message = parsed.error;
+                else if (parsed.message != null && parsed.message.length > 0)
+                    message = parsed.message;
+            }
+            catch { /* ignore parse failure on error body */ }
+            throw new WebError(message, "WEB_PROVIDER_ERROR");
+        }
+        try {
+            return mapCrwResponse(request.url, (await response.json()));
+        }
+        catch (error) {
+            if (signal?.aborted === true || isAbortError(error))
+                throw aborted(signal, error);
+            if (error instanceof WebError)
+                throw error;
+            throw new WebError(`crw returned an unprocessable response: ${String(error)}`, "WEB_PROVIDER_ERROR", { cause: error });
+        }
+    }
+}
+//#endregion
+//#region plugin
+/**
+ * Register a crw-backed fetch provider in `ctx.web`. crw is a Firecrawl-
+ * compatible crawl/scrape service. No auth, no API key.
+ *
+ * ## Usage
+ *
+ * ```yaml
+ * # cordis.patch.yml
+ * - insert:
+ *     - id: web-fetch-crw
+ *       name: '@jaco-tech/dsh-web-fetch-crw'
+ *       config:
+ *         baseURL: 'http://your-crw-instance:3000'
+ * ```
+ * @module @jaco-tech/dsh-web-fetch-crw
+ */
+/** Cordis plugin name used by loader diagnostics. */
+const name = "web-fetch-crw";
+/** The web seam this provider registers into. */
+const inject = ["web"];
+const Config = z.object({
+    baseURL: z.string().required(),
+});
+function apply(ctx, config) {
+    const current = () => config;
+    ctx.web.registerFetchProvider(new CrwFetchProvider(() => current()));
+}
+//#endregion
+//#region helpers
+function throwIfAborted(signal) {
+    if (signal?.aborted === true)
+        throw aborted(signal);
+}
+function aborted(signal, fallback) {
+    return new WebError("crw fetch aborted", "WEB_ABORTED", { cause: signal?.aborted === true ? signal.reason : fallback });
+}
+function isAbortError(error) {
+    return error instanceof DOMException && error.name === "AbortError";
+}
+//#endregion
+export { CRW_PROVIDER_ID, CrwFetchProvider, Config, apply, inject, name, mapCrwResponse };

package/lib/types/index.d.ts

@@ -0,0 +1,79 @@
+import z from "@deepseek-ai/schemastery";
+import type { Context } from "@deepseek-ai/cordis";
+import type { WebFetchProvider, WebFetchRequest, WebFetchResult } from "@deepseek-ai/dsh-web";
+interface CrwScrapeResponse {
+    success: boolean;
+    data?: {
+        markdown?: string;
+        raw?: string;
+        metadata?: {
+            sourceURL?: string;
+            statusCode?: number;
+            title?: string;
+            description?: string;
+            language?: string;
+        };
+        truncated?: boolean;
+        creditCost?: number;
+    };
+    error?: string;
+    message?: string;
+    warning?: string;
+    warnings?: string[];
+}
+/**
+ * Stable id this fetch provider registers under.
+ */
+declare const CRW_PROVIDER_ID = "crw";
+/**
+ * Map a crw scrape response to the seam's normalized `WebFetchResult`.
+ * crw returns markdown through `data.markdown` (Firecrawl-compatible API).
+ */
+declare function mapCrwResponse(url: string, body: CrwScrapeResponse): WebFetchResult;
+/**
+ * The crw-backed fetch provider. Each call issues a POST to `/v1/scrape`
+ * with the target URL and receives markdown content back.
+ *
+ * crw is a Firecrawl-compatible crawl/scrape service. No auth, no API key.
+ */
+declare class CrwFetchProvider implements WebFetchProvider {
+    readonly resolveOptions: () => {
+        baseURL: string;
+    };
+    readonly id = "crw";
+    constructor(resolveOptions: () => {
+        baseURL: string;
+    });
+    available(): boolean;
+    fetch(request: WebFetchRequest, signal?: AbortSignal): Promise<WebFetchResult>;
+}
+/**
+ * Register a crw-backed fetch provider in `ctx.web`. crw is a Firecrawl-
+ * compatible crawl/scrape service. No auth, no API key.
+ *
+ * ## Usage
+ *
+ * ```yaml
+ * # cordis.patch.yml
+ * - insert:
+ *     - id: web-fetch-crw
+ *       name: '@jaco-tech/dsh-web-fetch-crw'
+ *       config:
+ *         baseURL: 'http://your-crw-instance:3000'
+ * ```
+ * @module @jaco-tech/dsh-web-fetch-crw
+ */
+/** Cordis plugin name used by loader diagnostics. */
+declare const name = "web-fetch-crw";
+/** The web seam this provider registers into. */
+declare const inject: string[];
+/**
+ * Plugin configuration. `baseURL` is required (point at your crw instance).
+ */
+interface Config {
+    /** Base URL of the crw service (without /v1/scrape suffix). */
+    baseURL: string;
+}
+declare const Config: z<Config>;
+declare function apply(ctx: Context, config: Config): void;
+export { CRW_PROVIDER_ID, CrwFetchProvider, Config, apply, inject, name, mapCrwResponse };

package/package.json

@@ -0,0 +1,51 @@
+{
+  "name": "@jaco-tech/dsh-web-fetch-crw",
+  "description": "crw (Firecrawl-compatible) fetch provider for the DeepSeek Harness web capability seam (ctx.web). No auth, no API key.",
+  "version": "0.1.0",
+  "publishConfig": {
+    "access": "public"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/jaco-tech/dsh-web-fetch-crw.git"
+  },
+  "type": "module",
+  "main": "lib/index.js",
+  "types": "lib/types/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./lib/types/index.d.ts",
+      "default": "./lib/index.js"
+    },
+    "./package.json": "./package.json"
+  },
+  "files": [
+    "lib/index.js",
+    "lib/types/**/*.d.ts"
+  ],
+  "license": "MIT",
+  "keywords": [
+    "dsh-plugin",
+    "deepseek-harness",
+    "firecrawl",
+    "web-fetch",
+    "crawler",
+    "cordis-plugin"
+  ],
+  "peerDependencies": {
+    "@deepseek-ai/dsh-web": "^0.1.0-rc.7",
+    "@deepseek-ai/cordis": "^4.0.1"
+  },
+  "dependencies": {
+    "@deepseek-ai/schemastery": "^3.18.1"
+  },
+  "devDependencies": {
+    "@deepseek-ai/dsh-web": "^0.1.0-rc.7",
+    "@deepseek-ai/cordis": "^4.0.1",
+    "typescript": "^5.7.0"
+  },
+  "scripts": {
+    "build": "tsc",
+    "prepublishOnly": "tsc"
+  }
+}