dsh-ab-wechat-scrape 0.4.0

package/lib/types.d.ts

@@ -0,0 +1,133 @@
+/**
+ * Vocabulary shared by the wechat_scrape tool and its tests: the canonical
+ * value one call records, the call's own arguments, and the parsed page facts
+ * the extractor produces.
+ * @module @deepseek-ai/dsh-ab-wechat-scrape/types
+ */
+/** Body formats one call may ask for. */
+export type WechatFormat = 'markdown' | 'text';
+/** Arguments the wechat_scrape parameter schema admits. */
+export interface WechatArgs {
+    /** One official-account link: an article, or a list page when list is set. */
+    url?: string;
+    /** Several links, scraped in order and saved as separate files. */
+    urls?: string[];
+    /** Whether each link is a list or collection page whose articles are scraped instead of the page itself. */
+    list?: boolean;
+    /** Body format; markdown keeps headings, lists, links, and images. */
+    format?: WechatFormat;
+    /** Whether the value carries the article's image URLs. */
+    includeImages?: boolean;
+}
+/**
+ * One article after parsing, promotion filtering, and the deployment ceilings.
+ *
+ * This is what the call records and the model reads: the facts about the article,
+ * the length of its body, and a bounded head of it. The body itself is not here —
+ * it is in the file this article was saved to, which {@link WechatArticle.file}
+ * names.
+ */
+export interface CollectedArticle {
+    /** The link the model supplied. */
+    url: string;
+    /** The article link after any interstitial redirect was resolved. */
+    resolvedUrl: string;
+    /** Article title. */
+    title: string;
+    /** Official-account display name. */
+    account: string;
+    /** Author byline, empty when the page carries none. */
+    author: string;
+    /** Publish time as ISO 8601, or the page's own text when it is not a timestamp. */
+    publishTime: string;
+    /** Article summary shown under the title. */
+    digest: string;
+    /** Cover image URL, empty when the page carries none. */
+    cover: string;
+    /** Characters in the whole body, which is in the saved file. */
+    chars: number;
+    /** Head of the body, capped by the deployment's excerpt ceiling. */
+    excerpt: string;
+    /** Whether the excerpt is shorter than the body. */
+    excerptClipped: boolean;
+    /** Content image URLs of the filtered body, in document order. */
+    images: string[];
+    /** Whether a browser render supplied the HTML. */
+    rendered: boolean;
+}
+/**
+ * One article as it is built, before its body is written and dropped.
+ *
+ * The body exists only between parsing and the write. It never reaches the
+ * canonical value, the model, or the session log: a call that carried it would pay
+ * the article's full length in context and then have it discarded again.
+ */
+export interface DraftedArticle extends CollectedArticle {
+    /** The whole body, in the requested format. Written to the file and nowhere else. */
+    body: string;
+}
+/** One article the call scraped and saved. */
+export interface WechatArticle extends CollectedArticle {
+    /** Absolute path of the Markdown file this call wrote for the article. */
+    file: string;
+}
+/** One link the call could not scrape. */
+export interface WechatFailure {
+    /** The link that failed. */
+    url: string;
+    /** Why it failed, as the tool reported it. */
+    error: string;
+}
+/** One list or collection page the call walked. */
+export interface WechatList {
+    /** The list page the model supplied. */
+    url: string;
+    /** The page's own title, empty when it carried none. */
+    title: string;
+    /** Distinct article links the walk produced, in page order. */
+    links: string[];
+    /** Whether the collection carried more entries than the call's ceilings allowed. */
+    truncated: boolean;
+}
+/** Canonical value one wechat_scrape call records. */
+export interface WechatScrape {
+    /** Articles that were retrieved and saved, in request order. */
+    articles: WechatArticle[];
+    /** List pages the call walked, in request order. */
+    lists: WechatList[];
+    /** Links that were requested but produced no article. */
+    failures: WechatFailure[];
+}
+/** One parsed article page, before promotion filtering and the deployment ceilings. */
+export interface ExtractedArticle {
+    /** Article title. */
+    title: string;
+    /** Official-account display name. */
+    account: string;
+    /** Author byline. */
+    author: string;
+    /** Publish time, already normalized when the page carries a timestamp. */
+    publishTime: string;
+    /** Article summary. */
+    digest: string;
+    /** Cover image URL. */
+    cover: string;
+    /** Inner HTML of the article body element, promotional blocks included. */
+    bodyHtml: string;
+    /** Content image URLs in document order. */
+    images: string[];
+}
+/** What parsing one page produced. */
+export type Extraction = {
+    kind: 'article';
+    article: ExtractedArticle;
+} | {
+    kind: 'blocked';
+    marker: string;
+} | {
+    kind: 'unavailable';
+    reason: string;
+} | {
+    kind: 'empty';
+};
+//# sourceMappingURL=types.d.ts.map

package/lib/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,yCAAyC;AACzC,MAAM,MAAM,YAAY,GAAG,UAAU,GAAG,MAAM,CAAA;AAE9C,2DAA2D;AAC3D,MAAM,WAAW,UAAU;IACzB,8EAA8E;IAC9E,GAAG,CAAC,EAAE,MAAM,CAAA;IACZ,mEAAmE;IACnE,IAAI,CAAC,EAAE,MAAM,EAAE,CAAA;IACf,4GAA4G;IAC5G,IAAI,CAAC,EAAE,OAAO,CAAA;IACd,sEAAsE;IACtE,MAAM,CAAC,EAAE,YAAY,CAAA;IACrB,0DAA0D;IAC1D,aAAa,CAAC,EAAE,OAAO,CAAA;CACxB;AAED;;;;;;;GAOG;AACH,MAAM,WAAW,gBAAgB;IAC/B,mCAAmC;IACnC,GAAG,EAAE,MAAM,CAAA;IACX,qEAAqE;IACrE,WAAW,EAAE,MAAM,CAAA;IACnB,qBAAqB;IACrB,KAAK,EAAE,MAAM,CAAA;IACb,qCAAqC;IACrC,OAAO,EAAE,MAAM,CAAA;IACf,uDAAuD;IACvD,MAAM,EAAE,MAAM,CAAA;IACd,mFAAmF;IACnF,WAAW,EAAE,MAAM,CAAA;IACnB,6CAA6C;IAC7C,MAAM,EAAE,MAAM,CAAA;IACd,yDAAyD;IACzD,KAAK,EAAE,MAAM,CAAA;IACb,gEAAgE;IAChE,KAAK,EAAE,MAAM,CAAA;IACb,oEAAoE;IACpE,OAAO,EAAE,MAAM,CAAA;IACf,oDAAoD;IACpD,cAAc,EAAE,OAAO,CAAA;IACvB,kEAAkE;IAClE,MAAM,EAAE,MAAM,EAAE,CAAA;IAChB,kDAAkD;IAClD,QAAQ,EAAE,OAAO,CAAA;CAClB;AAED;;;;;;GAMG;AACH,MAAM,WAAW,cAAe,SAAQ,gBAAgB;IACtD,qFAAqF;IACrF,IAAI,EAAE,MAAM,CAAA;CACb;AAED,8CAA8C;AAC9C,MAAM,WAAW,aAAc,SAAQ,gBAAgB;IACrD,0EAA0E;IAC1E,IAAI,EAAE,MAAM,CAAA;CACb;AAED,0CAA0C;AAC1C,MAAM,WAAW,aAAa;IAC5B,4BAA4B;IAC5B,GAAG,EAAE,MAAM,CAAA;IACX,8CAA8C;IAC9C,KAAK,EAAE,MAAM,CAAA;CACd;AAED,mDAAmD;AACnD,MAAM,WAAW,UAAU;IACzB,wCAAwC;IACxC,GAAG,EAAE,MAAM,CAAA;IACX,wDAAwD;IACxD,KAAK,EAAE,MAAM,CAAA;IACb,+DAA+D;IAC/D,KAAK,EAAE,MAAM,EAAE,CAAA;IACf,oFAAoF;IACpF,SAAS,EAAE,OAAO,CAAA;CACnB;AAED,sDAAsD;AACtD,MAAM,WAAW,YAAY;IAC3B,gEAAgE;IAChE,QAAQ,EAAE,aAAa,EAAE,CAAA;IACzB,oDAAoD;IACpD,KAAK,EAAE,UAAU,EAAE,CAAA;IACnB,yDAAyD;IACzD,QAAQ,EAAE,aAAa,EAAE,CAAA;CAC1B;AAED,uFAAuF;AACvF,MAAM,WAAW,gBAAgB;IAC/B,qBAAqB;IACrB,KAAK,EAAE,MAAM,CAAA;IACb,qCAAqC;IACrC,OAAO,EAAE,MAAM,CAAA;IACf,qBAAqB;IACrB,MAAM,EAAE,MAAM,CAAA;IACd,0EAA0E;IAC1E,WAAW,EAAE,MAAM,CAAA;IACnB,uBAAuB;IACvB,MAAM,EAAE,MAAM,CAAA;IACd,uBAAuB;IACvB,KAAK,EAAE,MAAM,CAAA;IACb,2EAA2E;IAC3E,QAAQ,EAAE,MAAM,CAAA;IAChB,4CAA4C;IAC5C,MAAM,EAAE,MAAM,EAAE,CAAA;CACjB;AAED,sCAAsC;AACtC,MAAM,MAAM,UAAU,GAClB;IAAE,IAAI,EAAE,SAAS,CAAC;IAAC,OAAO,EAAE,gBAAgB,CAAA;CAAE,GAC9C;IAAE,IAAI,EAAE,SAAS,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GACnC;IAAE,IAAI,EAAE,aAAa,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GACvC;IAAE,IAAI,EAAE,OAAO,CAAA;CAAE,CAAA"}

package/lib/types.js

@@ -0,0 +1,8 @@
+/**
+ * Vocabulary shared by the wechat_scrape tool and its tests: the canonical
+ * value one call records, the call's own arguments, and the parsed page facts
+ * the extractor produces.
+ * @module @deepseek-ai/dsh-ab-wechat-scrape/types
+ */
+export {};
+//# sourceMappingURL=types.js.map

package/lib/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;;;;GAKG"}

package/package.json

@@ -0,0 +1,91 @@
+{
+  "name": "dsh-ab-wechat-scrape",
+  "version": "0.4.0",
+  "type": "module",
+  "main": "lib/index.js",
+  "types": "lib/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./lib/index.d.ts",
+      "default": "./lib/index.js"
+    },
+    "./package.json": "./package.json"
+  },
+  "files": [
+    "lib/**/*",
+    "cordis.patch.yml",
+    "README.md",
+    "LICENSE",
+    "tsconfig.json",
+    "tsdown.config.ts"
+  ],
+  "scripts": {
+    "build": "tsc -p tsconfig.json && tsdown",
+    "clean": "rimraf lib",
+    "typecheck": "tsc -p tsconfig.json --noEmit",
+    "test": "node --test",
+    "lint": "publint && attw --pack . --profile esm-only",
+    "format": "prettier --write \"src/**/*.{ts,mjs}\" \"tests/**/*.mjs\" \"*.{json,yml,md}\"",
+    "verify": "npm run typecheck && npm run test && npm run build && npm run lint",
+    "prepublishOnly": "npm run verify",
+    "prepack": "npm run build",
+    "release": "npm publish --access public --no-git-checks"
+  },
+  "dsh": {
+    "bundle": {
+      "patch": "./cordis.patch.yml"
+    }
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/deepseek-ai/deepseek-harness.git",
+    "directory": ".dsh/plugins/dsh-ab-wechat-scrape"
+  },
+  "bugs": {
+    "url": "https://github.com/deepseek-ai/deepseek-harness/issues"
+  },
+  "homepage": "https://github.com/deepseek-ai/deepseek-harness/tree/main/.dsh/plugins/dsh-ab-wechat-scrape",
+  "license": "MIT",
+  "keywords": [
+    "deepseek-harness",
+    "dsh-plugin",
+    "cordis-plugin",
+    "wechat",
+    "wechat-scrape",
+    "scraper"
+  ],
+  "engines": {
+    "node": ">=18"
+  },
+  "dependencies": {
+    "@deepseek-ai/dsh-tools": "^0.1.7-rc.1",
+    "@deepseek-ai/schemastery": "^3.18.4"
+  },
+  "peerDependencies": {
+    "@deepseek-ai/cordis": "*"
+  },
+  "devDependencies": {
+    "@deepseek-ai/cordis": "^4.0.4",
+    "@deepseek-ai/cordis-plugin-include": "^1.0.9",
+    "@deepseek-ai/cordis-plugin-loader": "^1.0.5",
+    "@deepseek-ai/dsh-fs": "^0.1.7-rc.1",
+    "@deepseek-ai/dsh-fs-local": "^0.1.7-rc.1",
+    "@deepseek-ai/dsh-fs-sandbox": "^0.1.7-rc.1",
+    "@deepseek-ai/dsh-sandbox": "^0.1.7-rc.1",
+    "@deepseek-ai/dsh-sandbox-policy": "^0.1.7-rc.1",
+    "@deepseek-ai/dsh-system-prompt": "^0.1.7-rc.1",
+    "@types/node": "^22.10.2",
+    "tsdown": "^0.22.2",
+    "typescript": "^6.0.3",
+    "rimraf": "^6.0.1",
+    "publint": "^0.2.0",
+    "@arethetypeswrong/cli": "^0.17.0",
+    "prettier": "^3.3.0"
+  },
+  "optionalDependencies": {
+    "playwright": "^1.61.1"
+  }
+}

package/tsconfig.json

@@ -0,0 +1,30 @@
+{
+  "compilerOptions": {
+    "target": "es2024",
+    "module": "esnext",
+    "moduleResolution": "bundler",
+    "declaration": true,
+    "declarationMap": true,
+    "sourceMap": true,
+    "skipLibCheck": true,
+    "esModuleInterop": true,
+    "allowImportingTsExtensions": true,
+    "rewriteRelativeImportExtensions": true,
+    "strict": true,
+    "noUncheckedIndexedAccess": true,
+    "exactOptionalPropertyTypes": true,
+    "noImplicitOverride": true,
+    "noFallthroughCasesInSwitch": true,
+    "noUnusedLocals": true,
+    "noUnusedParameters": true,
+    "jsx": "react-jsx",
+    "types": [
+      "node"
+    ],
+    "rootDir": "src",
+    "outDir": "lib"
+  },
+  "include": [
+    "src"
+  ]
+}

package/tsdown.config.ts

@@ -0,0 +1,18 @@
+/** Bundle config: one node entry (host tool). */
+import { defineConfig } from 'tsdown'
+
+export default defineConfig({
+  name: "dsh-ab-wechat-scrape",
+  entry: ['lib/index.js'],
+  outDir: 'lib',
+  format: ['esm'],
+  platform: 'node',
+  target: 'es2024',
+  fixedExtension: false,
+  dts: false,
+  clean: false,
+  deps: {
+    neverBundle: [/^@deepseek-ai\//],
+    alwaysBundle: (specifier) => !specifier.startsWith('@deepseek-ai/') && !specifier.startsWith('node:'),
+  },
+})